def result_upload(toktok, gkey, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "upload-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' file = request.files['file'] fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' if ',' in fnam: return "INVALID, no ',' allowed in filenames" # Ensures no commands within the filename new_name = secure_filename(fnam) if not os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))): os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) bf.greyfish_log(IP_addr, toktok, "upload", "single file", '/'.join(DIR.split('++')), new_name) file.save( os.path.join( GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')), new_name)) return 'File succesfully uploaded to Greyfish'
def grey_dir(gkey, toktok, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "download-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join( DIR.split('++')) + '/' if not os.path.exists(USER_DIR): return 'INVALID, Directory not available' os.chdir(USER_DIR) tar = tarfile.open("summary.tar.gz", "w:gz") for ff in os.listdir('.'): tar.add(ff) tar.close() os.chdir(CURDIR) bf.greyfish_log(IP_addr, toktok, "download", "dir", '/'.join(DIR.split('++'))) return send_file(USER_DIR + "summary.tar.gz")
def delete_user(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-user") return "INVALID key, cannot create a new user" try: shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok)) if bf.influx_logs: hugo.write_points([{ "measurement": "delete_account", "tags": { "id": toktok, }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) return "User files and data have been completely deleted" except: return "User does not exist"
def all_usernames(): if not request.is_json: return "INVALID: Request is not json" proposal = request.get_json() # Checks the required fields # self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used req_fields = ["key", "self_ID"] req_check = bf.l2_contains_l1(req_fields, proposal.keys()) if req_check != []: return "INVALID: Lacking the following json fields to be read: " + ",".join( [str(a) for a in req_check]) gkey = proposal["key"] self_ID = proposal["self_ID"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, self_ID): bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames") return "INVALID key" bf.greyfish_admin_log(IP_addr, self_ID, "Get all usernames") # Checks the number of subdirectories, one assigned per user return ','.join([ f.path.replace(GREYFISH_FOLDER + "DIR_", '') for f in os.scandir(GREYFISH_FOLDER) if f.is_dir() ])
def create_user(toktok, gkey): # Gets the IP address IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): # Records all failed logins bf.failed_login(gkey, IP_addr, toktok, "create-new-user") return "INVALID key, cannot create a new user" try: os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok)) if bf.influx_logs: hugo.write_points([{ "measurement": "signup", "tags": { "id": toktok, }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) return "Greyfish cloud storage now available" except: return "User already has an account"
def all_user_files(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "json-all-user-files") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' bf.greyfish_log(IP_addr, toktok, "json summary", "all files") return jsonify(bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok))
def delete_checksum_file(toktok, gkey, FILE): IP_addr = request.environ['REMOTE_ADDR'] checksum_dir = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/checksum_files/' if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-file") return "INVALID key" if not os.path.exists(checksum_dir + FILE): return 'Checksum file is not present in Greyfish' os.remove(checksum_dir + FILE) bf.greyfish_log(IP_addr, toktok, "delete", "checksum file", FILE) return 'Checksum file succesfully deleted from Greyfish storage'
def upload_dir(gkey, toktok, DIR): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "upload-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' file = request.files['file'] fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' if ',' in fnam: return "INVALID, no ',' allowed in filenames" # Untars the file, makes a directory if it does not exist if ('.tar.gz' not in fnam) and ('.tgz' not in fnam): return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz' new_name = secure_filename(fnam) try: if os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))): shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) file.save( os.path.join( GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')), new_name)) tar = tarfile.open(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + new_name) tar.extractall(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) tar.close() os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + new_name) except: return "Could not open tar file" bf.greyfish_log(IP_addr, toktok, "upload", "dir", '/'.join(DIR.split('++'))) return 'Directory succesfully uploaded to Greyfish'
def create_user(): if not request.is_json: return "POST parameters could not be parsed" ppr = request.get_json() [error_occurs, missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"], ppr.keys()) if error_occurs: return "INVALID: Lacking the following json fields to be read: " + missing_fields toktok = ppr["user_id"] gkey = ppr["gkey"] # Gets the IP address IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): # Records all failed logins bf.failed_login(gkey, IP_addr, toktok, "create-new-user") return "INVALID key, cannot create a new user" user_action = bf.idb_writer('greyfish') # Stores usernames in Redis since this will be faster to check in the future r_users = redis.Redis(host=URL_BASE, password=os.environ['REDIS_AUTH'], db=5) if r_users.get(toktok) != None: return "User already has an account" try: user_action.write_points([{ "measurement": "user_action", "tags": { "id": toktok, "action": "signup" }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) r_users.set(toktok, "Active") return "Greyfish cloud storage now available" except: return "INVALID, Server Error: Could not connect to database"
def delete_dir(toktok, gkey, DIR): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-dir") return "INVALID key" try: shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/') bf.greyfish_log(IP_addr, toktok, "delete", "single dir", '/'.join(DIR.split('++'))) return "Directory deleted" except: return "User directory does not exist"
def delete_user(): if not request.is_json: return "POST parameters could not be parsed" ppr = request.get_json() [error_occurs, missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"], ppr.keys()) if error_occurs: return "INVALID: Lacking the following json fields to be read: " + missing_fields toktok = ppr["user_id"] gkey = ppr["gkey"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-user") return "INVALID key, cannot create a new user" user_action = bf.idb_writer('greyfish') r_users = redis.Redis(host=URL_BASE, password=os.environ['REDIS_AUTH'], db=5) if r_users.get(toktok) == None: return "User does not exist" try: user_action.write_points([{ "measurement": "user_action", "tags": { "id": toktok, "action": "delete account" }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) r_users.delete(toktok) return "User files and data have been completely deleted" except: return "INVALID, Server Error: Could not connect to database"
def push_all(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "push-all-user-content") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' try: file = request.files['file'] except: return "No file uploaded" fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' new_name = secure_filename(fnam) # Must be a valid tar file try: file.save(USER_DIR + new_name) tar = tarfile.open(USER_DIR + new_name) tar.getmembers() except: os.remove(USER_DIR + new_name) return "Tar file cannot be opened, must be .tgz or .tar.gz" user_data = [USER_DIR + x for x in os.listdir(USER_DIR) if x != fnam] # Deletes all current data and untars the new files for content in user_data: if os.path.isdir(content): shutil.rmtree(content) continue os.remove(content) tar.extractall(USER_DIR) tar.close() os.remove(USER_DIR + new_name) bf.greyfish_log(IP_addr, toktok, "push") return 'User contents updated in Greyfish'
def purge_olderthan(Xsec): if not request.is_json: return "INVALID: Request is not json" proposal = request.get_json() # Checks the required fields # self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used req_fields = ["key", "self_ID"] req_check = bf.l2_contains_l1(req_fields, proposal.keys()) if req_check != []: return "INVALID: Lacking the following json fields to be read: " + ",".join( [str(a) for a in req_check]) gkey = proposal["key"] self_ID = proposal["self_ID"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, self_ID): bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames") return "INVALID key" try: XS = float(Xsec) except: return Xsec + " cannot be transformed into a float" # Gets the list of user directories user_dirs = [f.path for f in os.scandir(GREYFISH_FOLDER) if f.is_dir()] now = time.time() files_deleted = 0 # Purges empty files for udir in user_dirs: for ff in glob.iglob(udir + "/**/*", recursive=True): if (os.stat(ff).st_mtime < (now - XS)) and (os.path.isfile(ff)): os.remove(ff) files_deleted += 1 # Purges empty subdirectories for the users, does not delete the user directory itself rmf.remove_empty_dirs(udir, udir) bf.greyfish_admin_log(IP_addr, self_ID, "Purged old files", str(files_deleted)) return "Deleted " + str(files_deleted) + " files"
def grey_file(gkey, toktok, FIL, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "download-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join( DIR.split('++')) + '/' if str(FIL) not in os.listdir(USER_DIR): return 'INVALID, File not available' bf.greyfish_log(IP_addr, toktok, "download", "single file", '/'.join(DIR.split('++')), FIL) return send_file(USER_DIR + str(FIL), as_attachment=True)
def delete_file(toktok, gkey, FILE, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' try: os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + str(FILE)) bf.greyfish_log(IP_addr, toktok, "delete", "single file", '/'.join(DIR.split('++')), new_nam) return 'File succesfully deleted from Greyfish storage' except: return 'File is not present in Greyfish'
def user_files(toktok, gkey, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "json-user-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' # Accounts for users without a sandbox yet try: bf.greyfish_log(IP_addr, toktok, "json summary", "single dir", '/'.join(DIR.split('++'))) return jsonify( bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok + '/' + '/'.join(DIR.split('++')))) except: return 'Sandbox not set-up, create a sandbox first'
def download_checksum_dir(gkey, toktok, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "download-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join( DIR.split('++')) + '/' if not os.path.exists(USER_DIR): return 'INVALID, Directory not available' os.chdir(USER_DIR) tar = tarfile.open("summary.tar.gz", "w:gz") to_be_tarred = [x for x in os.listdir('.') if x != "summary.tar.gz"] for ff in to_be_tarred: tar.add(ff) os.remove(ff) tar.close() checksum8_name = ch.sha256_checksum("summary.tar.gz")[:8] + ".tar.gz" checksum_dir = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/checksum_files/' # If the temporary directory does not exist, it creates it if not os.path.isdir(checksum_dir): os.makedirs(checksum_dir) shutil.move("summary.tar.gz", checksum_dir + checksum8_name) os.chdir(CURDIR) bf.greyfish_log(IP_addr, toktok, "download checksum", "dir", '/'.join(DIR.split('++'))) return send_file(checksum_dir + checksum8_name, as_attachment=True)