def reef_upload(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, no file submitted' file = request.files['file'] # Avoids errors with users with no sandbox assigned try: os.listdir('/root/project/api/sandbox_files/DIR_' + str(toktok)) except: return 'User sandbox is not set-up, create a sandbox first' # No user can have more than 2 Gb assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824): return 'User has exceded asssigned allocation. Max. allocation is ' + str( assigned_allocation) + ' GB' # Avoids empty filenames and those with commas if file.filename == '': return 'Invalid, no file uploaded' if ',' in file.filename: return "ERROR: No ',' allowed in filenames" # Ensures no commands within the file new_name = secure_filename(file.filename) file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) return 'File succesfully uploaded to Reef'
def allocation_status(): if request.method != 'POST': return "INVALID, no data provided" if request.form['token'] == '': return "INVALID, no token provided" toktok = request.form["token"] if pp.token_test(toktok) == False: return 'Invalid token' used_space = pp.user_sandbox_size(str(toktok)) / 1073741824 assigned_allocation = r_alloc.get(toktok).decode('UTF-8') all_info = { 'Max. allocation': assigned_allocation + ' GB', 'Used space': str(used_space) + ' GB', 'Space available left': str((1 - used_space / float(assigned_allocation)) * 100) + '% allocation available' } return jsonify(all_info)
def reef_allocation_status(toktok): if pp.token_test(toktok) == False: return 'Invalid token' used_space = pp.user_sandbox_size(str(toktok))/1073741824 assigned_allocation = r.get(toktok).decode('UTF-8') all_info = {'Max. allocation': assigned_allocation+' GB', 'Used space': str(used_space)+' GB', 'Space available left': str((1 - used_space/float(assigned_allocation))*100)+'% allocation available'} return jsonify(all_info)
def reef_upload(toktok): if pp.token_test(toktok) == False: return 'INVALID token' if request.method != 'POST': return 'INVALID, no file submitted' file = request.files['file'] assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation*1073741824): return 'INVALID, User has exceded asssigned allocation. Max. available allocation is '+str(assigned_allocation)+' GB' # Avoids empty filenames and those with commas if file.filename == '': return 'INVALID, no file uploaded' if ',' in file.filename: return "INVALID, no ',' allowed in filenames" resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok, data={"filename":secure_filename(file.filename)}, files={"file": file.read()}) return resp.content
def midas(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, no file submitted' file = request.files['file'] try: ALL_USER_DATA = os.listdir( '/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok)) except: return 'User sandbox is not set-up, create a sandbox first' # If no app is provided, it will assume BOINC try: boapp = request.form["app"].lower() if (boapp != "boinc2docker") and (boapp != "adtdp"): return "INVALID app" except: boapp = "boinc2docker" # No user can submit jobs with a full allocation assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824): return 'User has exceded asssigned allocation. Current available allocation is ' + str( assigned_allocation) + ' GB' if file.filename == '': return 'Invalid, no file uploaded' if ',' in file.filename: return "ERROR: No ',' allowed in filenames" if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename): return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz' new_name = secure_filename(file.filename) file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) try: TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_name) except: return 'ERROR: python cannot open tar file' if not any('README.txt' in str(f) for f in TAR.getmembers()): os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) return 'ERROR: tar file does not contain mandatory README.txt' # Creates a new MIDAS directory with a new name while True: new_MID = 'MID_' + pp.random_dir_name() if new_MID not in ALL_USER_DATA: break # Checks the README for all necessary inputs TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID os.makedirs(TAR_PATH) TAR.extractall(TAR_PATH) if not mdr.valid_README(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: README is not valid' if not mdr.valid_OS(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: OS is not accepted' if not mdr.present_input_files(TAR_PATH): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Not all input files for commands are present' if not mdr.valid_language(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language is not accepted' # Avoids cases where no libraries are needed at all if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and ( mdr.install_libraries(TAR_PATH + '/README.txt') != []): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language does not support libraries' # Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp} r.set(toktok + ';' + new_MID, boapp) return 'File submitted for processing'
def delete_user_data(): if request.method != 'POST': return "INVALID, no data provided" if request.form['token'] == '': return "INVALID, no token provided" toktok = request.form["token"] if pp.token_test(toktok) == False: return "Invalid token" # All of the options must be selected with either y, or left blank # Sets up what to delete (more than one option may be selected): # - all: Deletes all the data in Reef, including directories # - basic: Deletes the temporary MIDAS directories, non-accessible to users in regular cases # - ordinary: Deletes Reef documents, not results, or MIDAS # - results: Deletes the results # Write y/Y/yes/YES to delete, all other commands will not delete delete_all = str(request.form['all']).lower() delete_basic = str(request.form['basic']).lower() delete_ordinary = str(request.form['ordinary']).lower() delete_results = str(request.form['results']).lower() user_commands = list( map(pp.y_parser, [delete_all, delete_basic, delete_ordinary, delete_results])) # Avoids users who do not want to delete any data if all((not x) for x in user_commands): return "INVALID, no files have been marked for erasing" # Computes the starting user space used_space = pp.user_sandbox_size(str(toktok)) / 1073741824 # Finds all the user data USER_PATH = "/root/project/api/sandbox_files/DIR_" + toktok user_files = os.listdir(USER_PATH) if user_commands[0]: # Calls all delete options user_commands = [True, True, True, True] if user_commands[1]: # Deletes all MIDAS directories ALL_MIDAS = [ USER_PATH + '/' + x for x in user_files if x[:4:] == "MID_" ] for y in ALL_MIDAS: shutil.rmtree(y) if user_commands[2]: # Deletes regular user files REGULAR_REEF = [ USER_PATH + '/' + x for x in user_files if (not os.path.isdir(USER_PATH + '/' + x)) ] for z in REGULAR_REEF: os.remove(z) if user_commands[3]: # Deletes the results RESULTS = os.listdir(USER_PATH + "/___RESULTS") for w in RESULTS: os.remove(USER_PATH + "/___RESULTS/" + w) # Computes the saved space now_space = pp.user_sandbox_size(str(toktok)) / 1073741824 recovered_space = used_space - now_space r_alloc.incrbyfloat(toktok, recovered_space) new_alloc = r_alloc.get(toktok).decode("UTF-8") return "Space recovered: " + str( recovered_space) + " GB; new allocated space: " + str( new_alloc) + " GB"