def result_upload(toktok, gkey, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "upload-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' file = request.files['file'] fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' if ',' in fnam: return "INVALID, no ',' allowed in filenames" # Ensures no commands within the filename new_name = secure_filename(fnam) if not os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))): os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) bf.greyfish_log(IP_addr, toktok, "upload", "single file", '/'.join(DIR.split('++')), new_name) file.save( os.path.join( GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')), new_name)) return 'File succesfully uploaded to Greyfish'
def grey_dir(gkey, toktok, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "download-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join( DIR.split('++')) + '/' if not os.path.exists(USER_DIR): return 'INVALID, Directory not available' os.chdir(USER_DIR) tar = tarfile.open("summary.tar.gz", "w:gz") for ff in os.listdir('.'): tar.add(ff) tar.close() os.chdir(CURDIR) bf.greyfish_log(IP_addr, toktok, "download", "dir", '/'.join(DIR.split('++'))) return send_file(USER_DIR + "summary.tar.gz")
def delete_user(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-user") return "INVALID key, cannot create a new user" try: shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok)) if bf.influx_logs: hugo.write_points([{ "measurement": "delete_account", "tags": { "id": toktok, }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) return "User files and data have been completely deleted" except: return "User does not exist"
def create_user(toktok, gkey): # Gets the IP address IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): # Records all failed logins bf.failed_login(gkey, IP_addr, toktok, "create-new-user") return "INVALID key, cannot create a new user" try: os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok)) if bf.influx_logs: hugo.write_points([{ "measurement": "signup", "tags": { "id": toktok, }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) return "Greyfish cloud storage now available" except: return "User already has an account"
def results_file(rkey, toktok, FIL): if not bf.valid_key(rkey): return "INVALID key" # user must be added to the database beforehand if not bf.valid_user(toktok): return "INVALID user" vmip, nkey = bf.get_file_vm(toktok, FIL, '') if vmip == None or nkey == None: return "INVALID, File not found" #req = requests.get("https://"+vmip+":3443"+"/reef/storage_reef/"+nkey+"/"+toktok+"/"+FIL) # https req = requests.get("https://" + vmip + ":3443" + "/reef/storage_reef/" + nkey + "/" + toktok + "/" + FIL, verify=False) # https with self signed certs #req = requests.get("http://"+vmip+":3443"+"/reef/storage_reef/"+nkey+"/"+toktok+"/"+FIL) # http if "INVALID" in req.text and len(req.text) < 40: return req.text USER_DIR = REEF_FOLDER + 'DIR_' + str(toktok) + '/download/' if not os.path.exists(USER_DIR): os.makedirs(USER_DIR) open(USER_DIR + FIL, 'wb').write(req.content) return send_file(USER_DIR + str(FIL))
def result_upload(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key" # user must be added to the database beforehand if not bf.valid_user(toktok): return "INVALID user" if request.method != 'POST': return 'INVALID, no file submitted' file = request.files['file'] # Avoids empty filenames and those with commas if file.filename == '': return 'INVALID, no file uploaded' if ',' in file.filename: return "INVALID, no ',' allowed in filenames" # delete file with the same name if already exist new_name = secure_filename(file.filename) filevm, vmkey = bf.get_file_vm(toktok, new_name, '___RESULTS') if filevm != None and vmkey != None: #delete = requests.get("https://"+filevm+":3443/reef/storage_delete_file/"+vmkey+"/"+toktok+"/"+new_name+"/___RESULTS") # https delete = requests.get( "https://" + filevm + ":3443/reef/storage_delete_file/" + vmkey + "/" + toktok + "/" + new_name + "/___RESULTS", verify=False) # https with self signed certificate #delete = requests.get("http://"+filevm+":3443/reef/storage_delete_file/"+vmkey+"/"+toktok+"/"+new_name+"/___RESULTS") # http #save file locally UPLOAD_DIR = REEF_FOLDER + 'DIR_' + str(toktok) + '/upload' if not os.path.exists(UPLOAD_DIR): os.makedirs(UPLOAD_DIR) file.save(os.path.join(UPLOAD_DIR, new_name)) # find VM that can fit the file filesize = os.stat(os.path.join(UPLOAD_DIR, new_name)).st_size vmip, nkey = bf.get_available_vms(filesize) if len(vmip) == 0 or len(nkey) == 0: os.remove(os.path.join(UPLOAD_DIR, new_name)) return "Couldn't find any VM which can fit the file to be uploaded" ip = vmip[0] key = nkey[0] # upload the file to the first available VM files = {'file': open(os.path.join(UPLOAD_DIR, new_name), 'rb')} #req = requests.post("https://"+ip+":3443"+"/reef/storage_upload/"+key+"/"+toktok+"/___RESULTS", files=files) # https req = requests.post("https://" + ip + ":3443" + "/reef/storage_upload/" + key + "/" + toktok + "/___RESULTS", files=files, verify=False) # https with self signed certificate #req = requests.post("http://"+ip+":3443"+"/reef/storage_upload/"+key+"/"+toktok+"/___RESULTS", files=files) # http # remove the file from local storage if os.path.exists(os.path.join(UPLOAD_DIR, new_name)): os.remove(os.path.join(UPLOAD_DIR, new_name)) return req.text
def all_usernames(): if not request.is_json: return "INVALID: Request is not json" proposal = request.get_json() # Checks the required fields # self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used req_fields = ["key", "self_ID"] req_check = bf.l2_contains_l1(req_fields, proposal.keys()) if req_check != []: return "INVALID: Lacking the following json fields to be read: " + ",".join( [str(a) for a in req_check]) gkey = proposal["key"] self_ID = proposal["self_ID"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, self_ID): bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames") return "INVALID key" bf.greyfish_admin_log(IP_addr, self_ID, "Get all usernames") # Checks the number of subdirectories, one assigned per user return ','.join([ f.path.replace(GREYFISH_FOLDER + "DIR_", '') for f in os.scandir(GREYFISH_FOLDER) if f.is_dir() ])
def all_user_files(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' return jsonify(bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok))
def reef_results_all(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key, cannot create a new user" USER_DIR = REEF_FOLDER + 'DIR_' + str(toktok) + '/___RESULTS' # Returns the results (space-separated) return ' '.join(os.listdir(USER_DIR))
def all_user_files(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key" AAA = [] AAA, user = bf.get_user_files(toktok) if len(AAA) == 0 and user == None: return "Sandbox not set-up, create a sandbox first" return ' '.join(AAA)
def all_user_files(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "json-all-user-files") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' bf.greyfish_log(IP_addr, toktok, "json summary", "all files") return jsonify(bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok))
def create_user(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key, cannot create a new user" try: os.makedirs(REEF_FOLDER+'DIR_'+str(toktok)) os.makedirs(REEF_FOLDER+'DIR_'+str(toktok)+'/___RESULTS') return "Reef cloud storage now available" except: return "User already has an account"
def results_file(rkey, toktok, FIL): if not bf.valid_key(rkey): return "INVALID key" if str('DIR_' + toktok) not in os.listdir(REEF_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = REEF_FOLDER + 'DIR_' + str(toktok) + '/' if str(FIL) not in os.listdir(USER_DIR): return 'INVALID, File not available' return send_file(USER_DIR + str(FIL))
def delete_checksum_file(toktok, gkey, FILE): IP_addr = request.environ['REMOTE_ADDR'] checksum_dir = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/checksum_files/' if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-file") return "INVALID key" if not os.path.exists(checksum_dir + FILE): return 'Checksum file is not present in Greyfish' os.remove(checksum_dir + FILE) bf.greyfish_log(IP_addr, toktok, "delete", "checksum file", FILE) return 'Checksum file succesfully deleted from Greyfish storage'
def delete_user(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): return "INVALID key, cannot create a new user" try: shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok)) return "User files and data have been completely deleted" except: return "User does not exist"
def delete_user_file(toktok, rkey, FILE): if not bf.valid_key(rkey): return "INVALID key" if str('DIR_' + toktok) not in os.listdir(REEF_FOLDER): return 'INVALID, User directory does not exist' try: os.remove(REEF_FOLDER + 'DIR_' + str(toktok) + '/' + str(FILE)) return 'File succesfully deleted from reef storage' except: return 'File is not present in Reef'
def reef_results_all(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key" AAA = [] AAA, user = bf.get_user_files(toktok, '___RESULTS') if len(AAA) == 0 and user == None: return 'Sandbox not set-up, create a sandbox first' # Returns the results (space-separated) return ' '.join(AAA)
def create_user(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key, cannot create a new user" # Stores usernames in MYSQL since this will be faster to check in the future grey_db = mysql_con.connect(host=os.environ["URL_BASE"], port=6602, user=os.environ["MYSQL_USER"], password=os.environ["MYSQL_PASSWORD"], database=os.environ["MYSQL_DATABASE"]) cursor = grey_db.cursor(buffered=True) cursor.execute("select * from user where name=%s", (toktok, )) uc = None for row in cursor: uc = row[0] cursor.close() grey_db.close() if uc != None: return "User already has an account" print('User doesn\'t exist') try: grey_db = mysql_con.connect(host=os.environ["URL_BASE"], port=6602, user=os.environ["MYSQL_USER"], password=os.environ["MYSQL_PASSWORD"], database=os.environ["MYSQL_DATABASE"]) cursor = grey_db.cursor(buffered=True) cursor.execute("insert into user(name) values(%s)", (toktok, )) grey_db.commit() cursor.execute("select ip from node") nodes = [] for row in cursor: nodes.append(row[0]) for node in nodes: cursor.execute( "insert into file set ip=%s, user_id=%s, id='', directory='', is_dir=TRUE", (node, toktok)) cursor.execute( "insert into file set ip=%s, user_id=%s, id='', directory='___RESULTS', is_dir=TRUE", (node, toktok)) grey_db.commit() cursor.close() grey_db.close() return "Reef cloud storage now available" except: traceback.print_exc() return "INVALID, Server Error: Could not connect to database"
def upload_dir(gkey, toktok, DIR): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "upload-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' file = request.files['file'] fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' if ',' in fnam: return "INVALID, no ',' allowed in filenames" # Untars the file, makes a directory if it does not exist if ('.tar.gz' not in fnam) and ('.tgz' not in fnam): return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz' new_name = secure_filename(fnam) try: if os.path.exists(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))): shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) file.save( os.path.join( GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')), new_name)) tar = tarfile.open(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + new_name) tar.extractall(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++'))) tar.close() os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + new_name) except: return "Could not open tar file" bf.greyfish_log(IP_addr, toktok, "upload", "dir", '/'.join(DIR.split('++'))) return 'Directory succesfully uploaded to Greyfish'
def create_user(): if not request.is_json: return "POST parameters could not be parsed" ppr = request.get_json() [error_occurs, missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"], ppr.keys()) if error_occurs: return "INVALID: Lacking the following json fields to be read: " + missing_fields toktok = ppr["user_id"] gkey = ppr["gkey"] # Gets the IP address IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): # Records all failed logins bf.failed_login(gkey, IP_addr, toktok, "create-new-user") return "INVALID key, cannot create a new user" user_action = bf.idb_writer('greyfish') # Stores usernames in Redis since this will be faster to check in the future r_users = redis.Redis(host=URL_BASE, password=os.environ['REDIS_AUTH'], db=5) if r_users.get(toktok) != None: return "User already has an account" try: user_action.write_points([{ "measurement": "user_action", "tags": { "id": toktok, "action": "signup" }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) r_users.set(toktok, "Active") return "Greyfish cloud storage now available" except: return "INVALID, Server Error: Could not connect to database"
def create_user(toktok, gkey): # Gets the IP address IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): # Records all failed logins return "INVALID key, cannot create a new user" try: os.makedirs(GREYFISH_FOLDER + 'DIR_' + str(toktok)) return "Greyfish cloud storage now available" except: return "User already has an account"
def delete_dir(toktok, gkey, DIR): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-dir") return "INVALID key" try: shutil.rmtree(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/') bf.greyfish_log(IP_addr, toktok, "delete", "single dir", '/'.join(DIR.split('++'))) return "Directory deleted" except: return "User directory does not exist"
def push_all(toktok, gkey): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "push-all-user-content") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' try: file = request.files['file'] except: return "No file uploaded" fnam = file.filename # Avoids empty filenames and those with commas if fnam == '': return 'INVALID, no file uploaded' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' new_name = secure_filename(fnam) # Must be a valid tar file try: file.save(USER_DIR + new_name) tar = tarfile.open(USER_DIR + new_name) tar.getmembers() except: os.remove(USER_DIR + new_name) return "Tar file cannot be opened, must be .tgz or .tar.gz" user_data = [USER_DIR + x for x in os.listdir(USER_DIR) if x != fnam] # Deletes all current data and untars the new files for content in user_data: if os.path.isdir(content): shutil.rmtree(content) continue os.remove(content) tar.extractall(USER_DIR) tar.close() os.remove(USER_DIR + new_name) bf.greyfish_log(IP_addr, toktok, "push") return 'User contents updated in Greyfish'
def delete_user(): if not request.is_json: return "POST parameters could not be parsed" ppr = request.get_json() [error_occurs, missing_fields] = bf.error__l2_contains_l1(["user_id", "gkey"], ppr.keys()) if error_occurs: return "INVALID: Lacking the following json fields to be read: " + missing_fields toktok = ppr["user_id"] gkey = ppr["gkey"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-user") return "INVALID key, cannot create a new user" user_action = bf.idb_writer('greyfish') r_users = redis.Redis(host=URL_BASE, password=os.environ['REDIS_AUTH'], db=5) if r_users.get(toktok) == None: return "User does not exist" try: user_action.write_points([{ "measurement": "user_action", "tags": { "id": toktok, "action": "delete account" }, "time": bf.timformat(), "fields": { "client-IP": IP_addr } }]) r_users.delete(toktok) return "User files and data have been completely deleted" except: return "INVALID, Server Error: Could not connect to database"
def purge_olderthan(Xsec): if not request.is_json: return "INVALID: Request is not json" proposal = request.get_json() # Checks the required fields # self_ID (str) refers to the self-identity of the user, only useful for checking with Redis in case a temporary token is used req_fields = ["key", "self_ID"] req_check = bf.l2_contains_l1(req_fields, proposal.keys()) if req_check != []: return "INVALID: Lacking the following json fields to be read: " + ",".join( [str(a) for a in req_check]) gkey = proposal["key"] self_ID = proposal["self_ID"] IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, self_ID): bf.failed_login(gkey, IP_addr, self_ID, "Get all usernames") return "INVALID key" try: XS = float(Xsec) except: return Xsec + " cannot be transformed into a float" # Gets the list of user directories user_dirs = [f.path for f in os.scandir(GREYFISH_FOLDER) if f.is_dir()] now = time.time() files_deleted = 0 # Purges empty files for udir in user_dirs: for ff in glob.iglob(udir + "/**/*", recursive=True): if (os.stat(ff).st_mtime < (now - XS)) and (os.path.isfile(ff)): os.remove(ff) files_deleted += 1 # Purges empty subdirectories for the users, does not delete the user directory itself rmf.remove_empty_dirs(udir, udir) bf.greyfish_admin_log(IP_addr, self_ID, "Purged old files", str(files_deleted)) return "Deleted " + str(files_deleted) + " files"
def grey_file(gkey, toktok, FIL, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "download-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' USER_DIR = GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join( DIR.split('++')) + '/' if str(FIL) not in os.listdir(USER_DIR): return 'INVALID, File not available' bf.greyfish_log(IP_addr, toktok, "download", "single file", '/'.join(DIR.split('++')), FIL) return send_file(USER_DIR + str(FIL), as_attachment=True)
def all_user_files(toktok, rkey): if not bf.valid_key(rkey): return "INVALID key" if str('DIR_' + toktok) not in os.listdir(REEF_FOLDER): return 'INVALID, User directory does not exist' # Accounts for users without a sandbox yet try: AAA = [] for afil in os.listdir(REEF_FOLDER + 'DIR_' + str(toktok)): AAA.append(afil) return ' '.join(AAA) except: return 'Sandbox not set-up, create a sandbox first'
def delete_file(toktok, gkey, FILE, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "delete-file") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' try: os.remove(GREYFISH_FOLDER + 'DIR_' + str(toktok) + '/' + '/'.join(DIR.split('++')) + '/' + str(FILE)) bf.greyfish_log(IP_addr, toktok, "delete", "single file", '/'.join(DIR.split('++')), new_nam) return 'File succesfully deleted from Greyfish storage' except: return 'File is not present in Greyfish'
def user_files(toktok, gkey, DIR=''): IP_addr = request.environ['REMOTE_ADDR'] if not bf.valid_key(gkey, toktok): bf.failed_login(gkey, IP_addr, toktok, "json-user-dir") return "INVALID key" if str('DIR_' + toktok) not in os.listdir(GREYFISH_FOLDER): return 'INVALID, User directory does not exist' # Accounts for users without a sandbox yet try: bf.greyfish_log(IP_addr, toktok, "json summary", "single dir", '/'.join(DIR.split('++'))) return jsonify( bf.structure_in_json(GREYFISH_FOLDER + 'DIR_' + toktok + '/' + '/'.join(DIR.split('++')))) except: return 'Sandbox not set-up, create a sandbox first'
def delete_user_file(toktok, rkey, FILE): if not bf.valid_key(rkey): return "INVALID key" # user must be added to the database beforehand if not bf.valid_user(toktok): return "INVALID user" vmip, nkey = bf.get_file_vm(toktok, FILE, '') if vmip == None or nkey == None: return "INVALID, file not found" #req = requests.get("https://"+vmip+":3443"+"/reef/storage_delete_file/"+nkey+"/"+toktok+"/"+FILE) # https req = requests.get("https://" + vmip + ":3443" + "/reef/storage_delete_file/" + nkey + "/" + toktok + "/" + FILE, verify=False) # https with self signed certs #req = requests.get("http://"+vmip+":3443"+"/reef/storage_delete_file/"+nkey+"/"+toktok+"/"+FILE) # http return req.text