def result_upload(toktok, gkey, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "upload-file")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
file = request.files['file']
fnam = file.filename
# Avoids empty filenames and those with commas
if fnam == '':
return 'INVALID, no file uploaded'
if ',' in fnam:
return "INVALID, no ',' allowed in filenames"
# Ensures no commands within the filename
new_name = secure_filename(fnam)
if not os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++'))):
os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')))
bf.greyfish_log(IP_addr, toktok, "upload", "single file",
'/'.join(DIR.split('++')), new_name)
file.save(
os.path.join(
GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')), new_name))
return 'File succesfully uploaded to Greyfish'
def grey_dir(gkey, toktok, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "download-dir")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(
DIR.split('++')) + '/'
if not os.path.exists(USER_DIR):
return 'INVALID, Directory not available'
os.chdir(USER_DIR)
tar = tarfile.open("summary.tar.gz", "w:gz")
for ff in os.listdir('.'):
tar.add(ff)
tar.close()
os.chdir(CURDIR)
bf.greyfish_log(IP_addr, toktok, "download", "dir",
'/'.join(DIR.split('++')))
return send_file(USER_DIR + "summary.tar.gz")
def delete_user(toktok, gkey):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "delete-user")
return "INVALID key, cannot create a new user"
try:
shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok))
if bf.influx_logs:
hugo.write_points([{
"measurement": "delete_account",
"tags": {
"id": toktok,
},
"time": bf.timformat(),
"fields": {
"client-IP": IP_addr
}
}])
return "User files and data have been completely deleted"
except:
return "User does not exist"
def all_usernames():
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
# self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used
req_fields = ["key", "self_ID"]
req_check = bf.l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: " + ",".join(
[str(a) for a in req_check])
gkey = proposal["key"]
self_ID = proposal["self_ID"]
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, self_ID):
bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames")
return "INVALID key"
bf.greyfish_admin_log(IP_addr, self_ID, "Get all usernames")
# Checks the number of subdirectories, one assigned per user
return ','.join([
f.path.replace(GREYFISH_FOLDER + "DIR_", '')
for f in os.scandir(GREYFISH_FOLDER) if f.is_dir()
])
def create_user(toktok, gkey):
# Gets the IP address
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
# Records all failed logins
bf.failed_login(gkey, IP_addr, toktok, "create-new-user")
return "INVALID key, cannot create a new user"
try:
os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok))
if bf.influx_logs:
hugo.write_points([{
"measurement": "signup",
"tags": {
"id": toktok,
},
"time": bf.timformat(),
"fields": {
"client-IP": IP_addr
}
}])
return "Greyfish cloud storage now available"
except:
return "User already has an account"
def all_user_files(toktok, gkey):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "json-all-user-files")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
bf.greyfish_log(IP_addr, toktok, "json summary", "all files")
return jsonify(bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok))
def delete_checksum_file(toktok, gkey, FILE):
IP_addr = request.environ['REMOTE_ADDR']
checksum_dir = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/checksum_files/'
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "delete-file")
return "INVALID key"
if not os.path.exists(checksum_dir + FILE):
return 'Checksum file is not present in Greyfish'
os.remove(checksum_dir + FILE)
bf.greyfish_log(IP_addr, toktok, "delete", "checksum file", FILE)
return 'Checksum file succesfully deleted from Greyfish storage'
def upload_dir(gkey, toktok, DIR):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "upload-dir")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
file = request.files['file']
fnam = file.filename
# Avoids empty filenames and those with commas
if fnam == '':
return 'INVALID, no file uploaded'
if ',' in fnam:
return "INVALID, no ',' allowed in filenames"
# Untars the file, makes a directory if it does not exist
if ('.tar.gz' not in fnam) and ('.tgz' not in fnam):
return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz'
new_name = secure_filename(fnam)
try:
if os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++'))):
shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')))
os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')))
file.save(
os.path.join(
GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')), new_name))
tar = tarfile.open(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')) + '/' + new_name)
tar.extractall(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')))
tar.close()
os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')) + '/' + new_name)
except:
return "Could not open tar file"
bf.greyfish_log(IP_addr, toktok, "upload", "dir",
'/'.join(DIR.split('++')))
return 'Directory succesfully uploaded to Greyfish'
def create_user():
if not request.is_json:
return "POST parameters could not be parsed"
ppr = request.get_json()
[error_occurs,
missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"],
ppr.keys())
if error_occurs:
return "INVALID: Lacking the following json fields to be read: " + missing_fields
toktok = ppr["user_id"]
gkey = ppr["gkey"]
# Gets the IP address
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
# Records all failed logins
bf.failed_login(gkey, IP_addr, toktok, "create-new-user")
return "INVALID key, cannot create a new user"
user_action = bf.idb_writer('greyfish')
# Stores usernames in Redis since this will be faster to check in the future
r_users = redis.Redis(host=URL_BASE,
password=os.environ['REDIS_AUTH'],
db=5)
if r_users.get(toktok) != None:
return "User already has an account"
try:
user_action.write_points([{
"measurement": "user_action",
"tags": {
"id": toktok,
"action": "signup"
},
"time": bf.timformat(),
"fields": {
"client-IP": IP_addr
}
}])
r_users.set(toktok, "Active")
return "Greyfish cloud storage now available"
except:
return "INVALID, Server Error: Could not connect to database"
def delete_dir(toktok, gkey, DIR):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "delete-dir")
return "INVALID key"
try:
shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')) + '/')
bf.greyfish_log(IP_addr, toktok, "delete", "single dir",
'/'.join(DIR.split('++')))
return "Directory deleted"
except:
return "User directory does not exist"
def delete_user():
if not request.is_json:
return "POST parameters could not be parsed"
ppr = request.get_json()
[error_occurs,
missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"],
ppr.keys())
if error_occurs:
return "INVALID: Lacking the following json fields to be read: " + missing_fields
toktok = ppr["user_id"]
gkey = ppr["gkey"]
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "delete-user")
return "INVALID key, cannot create a new user"
user_action = bf.idb_writer('greyfish')
r_users = redis.Redis(host=URL_BASE,
password=os.environ['REDIS_AUTH'],
db=5)
if r_users.get(toktok) == None:
return "User does not exist"
try:
user_action.write_points([{
"measurement": "user_action",
"tags": {
"id": toktok,
"action": "delete account"
},
"time": bf.timformat(),
"fields": {
"client-IP": IP_addr
}
}])
r_users.delete(toktok)
return "User files and data have been completely deleted"
except:
return "INVALID, Server Error: Could not connect to database"
def push_all(toktok, gkey):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "push-all-user-content")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
try:
file = request.files['file']
except:
return "No file uploaded"
fnam = file.filename
# Avoids empty filenames and those with commas
if fnam == '':
return 'INVALID, no file uploaded'
USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/'
new_name = secure_filename(fnam)
# Must be a valid tar file
try:
file.save(USER_DIR + new_name)
tar = tarfile.open(USER_DIR + new_name)
tar.getmembers()
except:
os.remove(USER_DIR + new_name)
return "Tar file cannot be opened, must be .tgz or .tar.gz"
user_data = [USER_DIR + x for x in os.listdir(USER_DIR) if x != fnam]
# Deletes all current data and untars the new files
for content in user_data:
if os.path.isdir(content):
shutil.rmtree(content)
continue
os.remove(content)
tar.extractall(USER_DIR)
tar.close()
os.remove(USER_DIR + new_name)
bf.greyfish_log(IP_addr, toktok, "push")
return 'User contents updated in Greyfish'
def purge_olderthan(Xsec):
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
# self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used
req_fields = ["key", "self_ID"]
req_check = bf.l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: " + ",".join(
[str(a) for a in req_check])
gkey = proposal["key"]
self_ID = proposal["self_ID"]
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, self_ID):
bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames")
return "INVALID key"
try:
XS = float(Xsec)
except:
return Xsec + " cannot be transformed into a float"
# Gets the list of user directories
user_dirs = [f.path for f in os.scandir(GREYFISH_FOLDER) if f.is_dir()]
now = time.time()
files_deleted = 0
# Purges empty files
for udir in user_dirs:
for ff in glob.iglob(udir + "/**/*", recursive=True):
if (os.stat(ff).st_mtime < (now - XS)) and (os.path.isfile(ff)):
os.remove(ff)
files_deleted += 1
# Purges empty subdirectories for the users, does not delete the user directory itself
rmf.remove_empty_dirs(udir, udir)
bf.greyfish_admin_log(IP_addr, self_ID, "Purged old files",
str(files_deleted))
return "Deleted " + str(files_deleted) + " files"
def grey_file(gkey, toktok, FIL, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "download-file")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(
DIR.split('++')) + '/'
if str(FIL) not in os.listdir(USER_DIR):
return 'INVALID, File not available'
bf.greyfish_log(IP_addr, toktok, "download", "single file",
'/'.join(DIR.split('++')), FIL)
return send_file(USER_DIR + str(FIL), as_attachment=True)
def delete_file(toktok, gkey, FILE, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "delete-file")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
try:
os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' +
'/'.join(DIR.split('++')) + '/' + str(FILE))
bf.greyfish_log(IP_addr, toktok, "delete", "single file",
'/'.join(DIR.split('++')), new_nam)
return 'File succesfully deleted from Greyfish storage'
except:
return 'File is not present in Greyfish'
def user_files(toktok, gkey, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "json-user-dir")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
# Accounts for users without a sandbox yet
try:
bf.greyfish_log(IP_addr, toktok, "json summary", "single dir",
'/'.join(DIR.split('++')))
return jsonify(
bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok + '/' +
'/'.join(DIR.split('++'))))
except:
return 'Sandbox not set-up, create a sandbox first'
def download_checksum_dir(gkey, toktok, DIR=''):
IP_addr = request.environ['REMOTE_ADDR']
if not bf.valid_key(gkey, toktok):
bf.failed_login(gkey, IP_addr, toktok, "download-dir")
return "INVALID key"
if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER):
return 'INVALID, User directory does not exist'
USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(
DIR.split('++')) + '/'
if not os.path.exists(USER_DIR):
return 'INVALID, Directory not available'
os.chdir(USER_DIR)
tar = tarfile.open("summary.tar.gz", "w:gz")
to_be_tarred = [x for x in os.listdir('.') if x != "summary.tar.gz"]
for ff in to_be_tarred:
tar.add(ff)
os.remove(ff)
tar.close()
checksum8_name = ch.sha256_checksum("summary.tar.gz")[:8] + ".tar.gz"
checksum_dir = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/checksum_files/'
# If the temporary directory does not exist, it creates it
if not os.path.isdir(checksum_dir):
os.makedirs(checksum_dir)
shutil.move("summary.tar.gz", checksum_dir + checksum8_name)
os.chdir(CURDIR)
bf.greyfish_log(IP_addr, toktok, "download checksum", "dir",
'/'.join(DIR.split('++')))
return send_file(checksum_dir + checksum8_name, as_attachment=True)
