def new_sandbox(): if request.method != 'POST': return 'Invalid, provide a token' TOK = request.form['token'] if pp.token_test(TOK) == False: return 'Invalid token' resp = requests.get('http://'+os.environ['Reef_IP']+':2002/reef/create_user/'+TOK+'/'+os.environ['Reef_Key']) return resp.text
def reef_results_all(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if str('DIR_' + toktok) not in os.listdir('/root/project/api/sandbox_files'): return 'User directory does not exist' USER_DIR = '/root/project/api/sandbox_files/DIR_' + str( toktok) + '/___RESULTS' # Returns the results (space-separated) return ' '.join(os.listdir(USER_DIR))
def obtain_file(FIL, toktok): if pp.token_test(toktok) == False: return 'Invalid token' if str('DIR_'+toktok) not in os.listdir('/root/project/api/sandbox_files'): return 'User directory does not exist' USER_DIR = '/root/project/api/sandbox_files/DIR_'+str(toktok)+'/' if str(FIL) not in os.listdir(USER_DIR): return 'File not available' return send_file(USER_DIR+str(FIL))
def user_images(toktok): if not pp.token_test(toktok): return 'Invalid token' user_images = {} for IMAGE in client.images.list(): try: NAME = IMAGE.attrs['RepoTags'][0] except: continue if NAME.split(':')[0] == toktok.lower(): user_images[NAME] = str(IMAGE.attrs['Size'] / (10**9)) + " GB" return jsonify(user_images)
def all_user_files(toktok): if pp.token_test(toktok) == False: return 'Invalid token' # Accounts for users without a sandbox yet try: AAA = [] for afil in os.listdir('/root/project/api/sandbox_files/DIR_'+str(toktok)): AAA.append(afil) return ','.join(AAA) except: return 'Sandbox not set-up, create a sandbox first'
def tacc_jobs(): # Ensures that there is an appropriate json request if not request.is_json: return "INVALID: Request is not json" proposal = request.get_json() if pp.token_test(proposal["token"]) == False: return 'Invalid token' # Checks the required fields req_fields = ["token", "image", "commands", "priority"] req_check = l2_contains_l1(req_fields, proposal.keys()) if req_check != []: return "INVALID: Lacking the following json fields to be read: " + ",".join( [str(a) for a in req_check]) [TOKEN, IMAGE, COMMANDS, PRIORITY] = [ proposal["token"], proposal["image"], proposal["commands"], proposal["priority"] ] VolCon_ID = uuid.uuid4().hex if "gpu" in IMAGE: GPU = 1 else: GPU = 0 try: mints.add_job(TOKEN, IMAGE, COMMANDS, GPU, VolCon_ID, PRIORITY) except: return "INVALID: Could not connect to MySQL database" # TACC: Image is a TACC image job_info = { "Image": IMAGE, "Command": COMMANDS, "TACC": 1, "GPU": GPU, "VolCon_ID": VolCon_ID, "public": 1 } mirror.upload_job_to_mirror(job_info) return "Successfully submitted job"
def delete_user_file(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, provide a file to be deleted' try: FILE = request.form['del'] if FILE == '': return 'No file provided' except: return "Cannot parse request, 'del' entry is not present" resp = requests.get('http://'+os.environ['Reef_IP']+':2000/reef/delete_file/'+os.environ['Reef_Key']+'/'+toktok+'/'+FILE) return resp.text
def reef_allocation_status(toktok): if pp.token_test(toktok) == False: return 'Invalid token' used_space = pp.user_sandbox_size(str(toktok)) / 1073741824 assigned_allocation = r.get(toktok).decode('UTF-8') all_info = { 'Max. allocation': assigned_allocation + ' GB', 'Used space': str(used_space) + ' GB', 'Space available left': str((1 - used_space / float(assigned_allocation)) * 100) + '% allocation available' } return jsonify(all_info)
def simple_allocation_check(): if request.method != 'POST': return "INVALID, no data provided" if request.form['token'] == '': return "INVALID, no token provided" toktok = request.form["token"] if pp.token_test(toktok) == False: return 'INVALID token' assigned_allocation = float(r_alloc.get(toktok).decode('UTF-8')) if assigned_allocation > 0: return 'y' return 'n'
def new_sandbox(): if request.method != 'POST': return 'Invalid, provide a token' TOK = request.form['token'] if pp.token_test(TOK) == False: return 'Invalid token' # Finds if the directory has already been created or not for sandir in os.listdir('/root/project/api/sandbox_files'): if TOK == sandir[4::]: return 'Sandbox already available' else: os.makedirs('/root/project/api/sandbox_files/DIR_' + str(TOK)) os.makedirs('/root/project/api/sandbox_files/DIR_' + str(TOK) + '/___RESULTS') return 'reef cloud storage now available'
def references(toktok): if pp.token_test(toktok) == False: return 'Invalid token' allref = { 'OS': { 'Ubuntu 16.04': 'Ubuntu_16.04' }, 'Languages': { 'python2': 'Not supported', 'python3': 'python, python3', 'Go': 'Go', 'R': 'R', 'Fortran': 'Fortran90', 'C': 'C', 'C++': 'C++' } } return jsonify(allref)
def delete_user_file(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, provide a file to be deleted' # Accounts for missing directories if str('DIR_'+toktok) not in os.listdir('/root/project/api/sandbox_files'): return 'User directory does not exist' try: FILE = request.form['del'] if FILE == '': return 'No file provided' os.remove('/root/project/api/sandbox_files/DIR_'+str(toktok)+'/'+str(FILE)) return 'File succesfully deleted from reef storage' except: return 'File is not present in Reef'
def reef_upload(toktok): if pp.token_test(toktok) == False: return 'INVALID token' if request.method != 'POST': return 'INVALID, no file submitted' file = request.files['file'] # Avoids empty filenames and those with commas if file.filename == '': return 'INVALID, no file uploaded' if ',' in file.filename: return "INVALID, no ',' allowed in filenames" resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok, data={"filename":secure_filename(file.filename)}, files={"file": file.read()}) return resp.content
def delete_midas_dir(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, provide a file to be deleted' # Accounts for missing directories if str('DIR_' + toktok) not in os.listdir( '/home/boincadm/project/api/sandbox_files'): return 'User directory does not exist' try: MIDIR = request.form['del'] if MIDIR == '': return 'No file provided' shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + str(MIDIR)) return 'Midas directory deleted' except: return 'MIDAS directory does not exist'
def allocation_status(): if request.method != 'POST': return "INVALID, no data provided" if request.form['token'] == '': return "INVALID, no token provided" toktok = request.form["token"] if pp.token_test(toktok) == False: return 'Invalid token' used_space = pp.user_sandbox_size(str(toktok))/1073741824 assigned_allocation = r_alloc.get(toktok).decode('UTF-8') all_info = {'Max. allocation': assigned_allocation+' GB', 'Used space': str(used_space)+' GB', 'Space available left': str((1 - used_space/float(assigned_allocation))*100)+'% allocation available'} return jsonify(all_info)
def upload_file(toktok): TOK = toktok if pp.token_test(TOK) == False: return 'Invalid token' if request.method != 'POST': return "Invalid. Submit a text file" file = request.files['file'] # If no app is provided, it will assume BOINC try: app = request.form["app"].lower() if (app != "boinc2docker") and (app != "adtdp"): return "INVALID app" except: app = "boinc2docker" # Avoids empty files and non-text files if file.filename == '': return 'No file submitted' if file.filename.split('.')[-1] != 'txt': return "File type invalid, only text files are acccepted" # Randomizes the file name to avoid matches new_filename = pp.random_file_name() file.save(os.path.join(UPLOAD_FOLDER, new_filename)) # Adds the token at the end of the file with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod: nod.write('\n' + str(TOK)) if app == "boinc2docker": shutil.move(UPLOAD_FOLDER + '/' + new_filename, FINAL_FOLDER + '/' + new_filename) if app == "adtdp": shutil.move(UPLOAD_FOLDER + '/' + new_filename, ADTDP_FOLDER + '/' + new_filename) return "File submitted for processing"
def add_username(username, email, toktok, org_key): org_key = hashlib.sha256(org_key.encode('UTF-8')).hexdigest()[:24:] all_org_keys = [ r_org.hmget(x.decode('UTF-8'), 'Organization Token')[0].decode('UTF-8') for x in r_org.keys() ] if org_key not in all_org_keys: return 'Organization key invalid, access denied' if pp.token_test(toktok) == False: return "INVALID token" for y in r_org.keys(): if org_key == r_org.hmget(y, 'Organization Token')[0].decode('UTF-8'): ORG = y.decode('UTF-8') break # If the user is already registered it does nothing for qq in range(0, r.llen(ORG)): if username == r.lindex(ORG, qq).decode("UTF-8"): break else: # Adds a new row for the user r.rpush(ORG, username) # Creates a new hash r.hmset(username, {email: toktok}) return "Added new username to the database" # Checks if the user has already set this email for zz in r.hkeys(username): if zz.decode('UTF-8') == email: return "User email already in database" # Adds another email to the user r.hset(username, email, toktok) return "Added another user email"
def delete_image(toktok): if not pp.token_test(toktok): return 'Invalid token' if request.method != 'POST': return 'Invalid, provide an image to be deleted' UTOK = str(toktok).lower() DATA = str(request.form['del']) IMTAG = DATA if ':' not in DATA: IMTAG = UTOK.lower() + ':' + DATA try: r.incrbyfloat(toktok, float(client.images.get(IMTAG).attrs['Size']) / (10**9)) client.images.remove(image=IMTAG, force=True) return 'User image has been deleted. User allocation is now ' + r.get( toktok).decode('UTF-8') + " GB" except: return 'ERROR: Image does not exist or is not owned by user'
def user_data(toktok): if pp.token_test(toktok) == False: return 'INVALID token' U_alloc = r_alloc.get(toktok).decode("UTF-8") # Finds all the data totjobs = r_data.llen("Token") U_data = [] for qq in range(0, totjobs): if r_data.lindex("Token", qq).decode("UTF-8") == toktok: # [[Image, Command, Date (Sub), Date (Run), Notified], ..] U_data.append([]) curdat = {} imnam = r_data.lindex("Image", qq).decode("UTF-8") if "carlosred/" != imnam[:10:]: imnam += " (CUSTOM)" curdat["Image"] = imnam curdat["Command"] = r_data.lindex("Command", qq).decode("UTF-8") curdat["Date (Sub)"] = transform_to_iso( r_data.lindex("Date (Sub)", qq).decode("UTF-8")) curdat["Date (Run)"] = transform_to_iso( r_data.lindex("Date (Run)", qq).decode("UTF-8")) curdat["Notified"] = transform_to_iso( r_data.lindex("Notified", qq).decode("UTF-8")) U_data[-1].append(curdat) # Appends the VolCon jobs vc_info = VolCon_info(toktok) U_data = U_data + vc_info # Sorts them by received date U_data = sorted(U_data, key=lambda x: x[0]["Date (Sub)"]) return jsonify({"allocation": U_alloc, "job data": U_data})
def reef_upload(toktok): if pp.token_test(toktok) == False: return 'INVALID token' if request.method != 'POST': return 'INVALID, no file submitted' file = request.files['file'] assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation*1073741824): return 'INVALID, User has exceded asssigned allocation. Max. available allocation is '+str(assigned_allocation)+' GB' # Avoids empty filenames and those with commas if file.filename == '': return 'INVALID, no file uploaded' if ',' in file.filename: return "INVALID, no ',' allowed in filenames" resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok, data={"filename":secure_filename(file.filename)}, files={"file": file.read()}) return resp.content
def reef_results_all(toktok): if pp.token_test(toktok) == False: return 'Invalid token' resp = requests.get('http://'+os.environ['Reef_IP']+':2001/reef/results_all/'+os.environ['Reef_Key']+'/'+toktok) return resp.text
def process_web_jobs(): if request.method != 'POST': return "INVALID, no data provided" try: dictdata = request.get_json() except: return "INVALID, JSON could not be parsed" try: TOK = dictdata["Token"] Reefil = dictdata["Files"] Image = dictdata["Image"] Custom = dictdata["Custom"] Command = dictdata["Command"] Username = dictdata["Username"] if "priority" not in dictdata.keys(): PRIORITY = "Middle" else: PRIORITY = dictdata["priority"] except: return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command, Username)" if pp.token_test(TOK) == False: return "INVALID token" # Checks the list of Commands to ensure their correct execution Command = ';'.join([C for C in Command.split(';') if (C != '') and (C.count(C[0]) != len(C)) ]) +';' boapp = "boinc2docker" if Image == "carlosred/gpu:cuda": boapp = "volcon" if (Custom != "Yes" ) and (Custom != "No"): return abort(422) # Incorrect API if not image_is_TACC(Image): Custom = "Yes" # Gets the tags try: tags_used = [x.strip() for x in dictdata["topics"].split(";") if x.strip() != ""] if tags_used == []: tags_used = "STEM" else: tags_used = ",".join(tags_used) tags_used = tags_used.lower() except Exception as e: print(e) # Error in processing json tags_used = "STEM" else: # Default tag: STEM tags_used = "STEM" if boapp == "boinc2docker": # Writes the commands to a random file new_filename = pp.random_file_name() Complete_command = "" # Custom images require more work because it must be ensured the results will be back if Custom == "Yes": # Creates a new working directory Complete_command += "mkdir -p /data; cd /data; " # Adds the files for FF in Reefil: # Skips unnecessary files if FF == "": break Complete_command += get_reef_file(Image, TOK, FF)+" " Complete_command += Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results" elif Custom == "No": # Adds the files for FF in Reefil: # Skips unnecessary files if FF == "": break Complete_command += get_reef_file(Image, TOK, FF)+" " Complete_command += Command +" mv ./* /root/shared/results" # Replaces certain characters Complete_command = " /bin/bash -c \""+sanitize_str_chars(Complete_command)+"\"" mints.add_boinc2docker_job(Username, TOK, tags_used, Image, Complete_command, boapp, "web", "Job submitted") # Old way # shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename) if boapp == "volcon": # Only CUDA requires a GPU # Custom images are also assumed to not require a GPU TODO TODO TODO TODO if Image == "carlosred/gpu:cuda": GPU = 1 else: GPU = 0 VolCon_ID = uuid.uuid4().hex COMMANDS = "" if Custom == "Yes": TACC = 0 COMMANDS += "mkdir -p /data; cd /data; " for FF in Reefil: if FF == '': break COMMANDS += get_reef_file(Image, TOK, FF)+" " # Splits the commands and ensures that they are run in /data newcoms = ";".join(["cd /data && "+x for x in Command.split(";")]) COMMANDS += newcoms+" mkdir -p /root/shared/results/; mv /data/* /root/shared/results" else: TACC = 1 for FF in Reefil: if FF == '': break COMMANDS += get_reef_file(Image, TOK, FF)+" " COMMANDS += ";".join([extra_image_commands(Image) +z for z in Command.split(";")])+" mv ./* /root/shared/results" COMMANDS = " /bin/bash -c \""+COMMANDS+"\"" job_info = {"Image":Image, "Command":COMMANDS, "TACC":TACC, "GPU":GPU, "VolCon_ID":VolCon_ID, "public":1} # Updates the job in the database mints.add_job(TOK, Image, COMMANDS, GPU, VolCon_ID, PRIORITY, 1, tags_used, Username, "web") # Pushes job information to mirrors mirror.upload_job_to_mirror(job_info) return "Commands submitted for processing"
def upload_file(toktok): TOK = toktok if pp.token_test(TOK) == False: return 'Invalid token' if request.method != 'POST': return "Invalid. Submit a text file" file = request.files['file'] # Gathers all the data try: dictdata = ImmutableMultiDict(request.form).to_dict(flat='') app = str(dictdata["app"][0].lower()) # Gets the list of topics/subtopics Tops = [j for j in dictdata.keys() if j != "app"] # Each topic will be defined by Topic: subtopics # Subtopics are separated by commas TST = {} # Topics structure for one_top in Tops: TST[one_top] = [ zz for zz in dictdata[one_top][0].upper().split(',') if zz != '' ] except: return "INVALID, not all data has been provided" # If no app is provided, it will assume BOINC try: if (app != "boinc2docker") and (app != "adtdp"): return "INVALID app" except: app = "boinc2docker" # Avoids empty files and non-text files if file.filename == '': return 'No file submitted' if file.filename.split('.')[-1] != 'txt': return "File type invalid, only text files are acccepted" # Randomizes the file name to avoid matches new_filename = pp.random_file_name() file.save(os.path.join(UPLOAD_FOLDER, new_filename)) # Gets the image names # Although uncommon, users can write multiple commands in a single file Im_Names = [] with open(UPLOAD_FOLDER + '/' + new_filename, 'r') as comfil: for line in comfil: LL = line.replace('\n', '') if len(line.split(' ')) == 1: continue Im_Names.append(line.split(' ')[0]) # Adds the token at the end of the file with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod: nod.write('\n' + str(TOK)) AA = '' if app == "boinc2docker": # Adds the topic to database for an_image in Im_Names: AA += cus.complete_tag_work(an_image, TST, jobsub=len(Im_Names)) shutil.move(UPLOAD_FOLDER + '/' + new_filename, FINAL_FOLDER + '/' + new_filename) if app == "adtdp": shutil.move(UPLOAD_FOLDER + '/' + new_filename, ADTDP_FOLDER + '/' + new_filename) return "File submitted for processing\n" + AA
def midas(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, no file submitted' file = request.files['file'] try: ALL_USER_DATA = os.listdir( '/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok)) except: return 'User sandbox is not set-up, create a sandbox first' # If no app is provided, it will assume BOINC try: boapp = request.form["app"].lower() if (boapp != "boinc2docker") and (boapp != "adtdp"): return "INVALID app" except: boapp = "boinc2docker" # No user can submit jobs with a full allocation assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824): return 'User has exceded asssigned allocation. Current available allocation is ' + str( assigned_allocation) + ' GB' if file.filename == '': return 'Invalid, no file uploaded' if ',' in file.filename: return "ERROR: No ',' allowed in filenames" if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename): return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz' new_name = secure_filename(file.filename) file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) try: TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_name) except: return 'ERROR: python cannot open tar file' if not any('README.txt' in str(f) for f in TAR.getmembers()): os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) return 'ERROR: tar file does not contain mandatory README.txt' # Creates a new MIDAS directory with a new name while True: new_MID = 'MID_' + pp.random_dir_name() if new_MID not in ALL_USER_DATA: break # Checks the README for all necessary inputs TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID os.makedirs(TAR_PATH) TAR.extractall(TAR_PATH) if not mdr.valid_README(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: README is not valid' if not mdr.valid_OS(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: OS is not accepted' if not mdr.present_input_files(TAR_PATH): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Not all input files for commands are present' if not mdr.valid_language(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language is not accepted' # Avoids cases where no libraries are needed at all if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and ( mdr.install_libraries(TAR_PATH + '/README.txt') != []): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language does not support libraries' # Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp} r.set(toktok + ';' + new_MID, boapp) return 'File submitted for processing'
def process_web_jobs(): if request.method != 'POST': return "INVALID, no data provided" try: dictdata = request.get_json() except: return "INVALID, JSON could not be parsed" try: TOK = dictdata["Token"] boapp = dictdata['Boapp'].lower() Reefil = dictdata["Files"] Image = dictdata["Image"] Custom = dictdata["Custom"] Command = dictdata["Command"] tprov = json.loads(dictdata["topics"]) # TACC images have already assigned topics if Image in cus.at.TACCIM.keys(): tprov = cus.at.TACCIM[Image] except: return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command)" if pp.token_test(TOK) == False: return "INVALID token" # Checks if user wants boinc2docker or adtd-p try: if (boapp != "boinc2docker") and (boapp != "adtdp"): return "INVALID application" except: return "INVALID, application not provided" if (Custom != "Yes") and (Custom != "No"): return "INVALID, Custom value can only be [Yes/No]" if ("Custom" == "No") and (not cus.image_is_TACC(Image)): return "INVALID, Image \'"+Image+"\' is not provided by TACC" # Writes the commands to a random file new_filename = pp.random_file_name() with open(UPLOAD_FOLDER+new_filename, "w") as comfil: comfil.write(Image + " /bin/bash -c ") # Custom images require more work because it must be ensured the results will be back if Custom == "Yes": # Creates a new working directory comfil.write("\"mkdir -p /data; cd /data; ") # Adds the files for FF in Reefil: comfil.write(get_reef_file(Image, TOK, FF)+" ") comfil.write(Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results\"") comfil.write("\n"+str(TOK)) elif Custom == "No": comfil.write("\"") comfil.write(extra_image_commands(Image)) # Adds the files for FF in Reefil: comfil.write(get_reef_file(Image, TOK, FF)+" ") comfil.write(Command+" python /Mov_Res.py\"") comfil.write("\n"+str(TOK)) # Submits the file for processing if boapp == "boinc2docker": toproc(Image, tprov) shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename) if boapp == "adtdp": shutil.move(UPLOAD_FOLDER+new_filename, ADTDP_FOLDER+new_filename) return "Commands submitted for processing"
def results_file(FIL, toktok): if pp.token_test(toktok) == False: return 'Invalid token' resp = requests.get('http://'+os.environ['Reef_IP']+':2001/reef/results/'+os.environ['Reef_Key']+'/'+toktok+'/'+FIL) return resp.content
def process_midas_jobs(): try: dictdata = request.get_json() except: return "INVALID, JSON could not be parsed" try: TOK = dictdata["token"] tmp_dir = dictdata["folder_name"] OS = dictdata["operating_system"] prolangs = dictdata["programming_language"] libs = dictdata["library_list"] setfiles = dictdata["setup_filename"] outfiles = dictdata["output_file"] coms = dictdata["command_lines"] except: return "INVALID, json lacks at least one field" if pp.token_test(TOK) == False: shutil.rmtree(TMP) return 'INVALID, invalid token' # MIDAS files are stored in a temporary folder boapp = "boinc2docker" TMP = "/tmp/" + tmp_dir + '/' if 'README.txt' in os.listdir(TMP): return "INVALID, README.txt is a MIDAS reserved file. Please, remove it and try again" for file in os.listdir(TMP): if (file.split(".")[-1] == "tgz") or (".".join( file.split(".")[::-1][:2:][::-1]) == "tar.gz"): # Takes the contents out and deletes the tar try: tar = tarfile.open(TMP + file) tar.extractall(TMP) tar.close() os.remove(TMP + file) continue except: shutil.rmtree(TMP) return "INVALID, cannot open tar file" if (file.split(".")[-1] == "zip"): try: zip_ref = zipfile.ZipFile(TMP + file, 'r') zip_ref.extractall(TMP) zip_ref.close() os.remove(TMP + file) except: shutil.rmtree(TMP) return "INVALID, cannot open zip file" # The application is adtdp if using c++ libraries if libs['cPlusPlus'] != []: boapp = "adtdp" # Creates an acceptable README with open(TMP + "README.txt", 'w') as readme: try: readme.write(verne(dictdata, TMP, boapp)) except Exception as e: shutil.rmtree(TMP) return "INVALID, " + str(e) # Validates if the file can be processed if not mdr.valid_README(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, README in user input' if not mdr.valid_OS(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, OS is not accepted' if not mdr.present_input_files(TMP): shutil.rmtree(TMP) return 'INVALID, Not all input files for commands are present' if not mdr.valid_language(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, Language is not accepted' # Avoids cases where no libraries are needed at all if (not mdr.install_libraries(TMP + '/README.txt')) and ( mdr.install_libraries(TMP + '/README.txt') != []): shutil.rmtree(TMP) return 'INVALID, Language does not support libraries' # Moves the directory to MIDAS and processes it ALL_USER_DATA = os.listdir('/root/project/api/sandbox_files/DIR_' + str(TOK)) while True: new_MID = 'MID_' + pp.random_dir_name() if new_MID not in ALL_USER_DATA: break MIDAS_PATH = UPLOAD_FOLDER + '/DIR_' + str(TOK) + '/' + new_MID shutil.copytree(TMP, MIDAS_PATH) shutil.rmtree(TMP) # Creates a redis database with syntax {TOKEN}.{MID_DIRECTORY} r.set(TOK + ';' + new_MID, boapp) return 'File submitted for processing'
def process_web_jobs(): if request.method != 'POST': return "INVALID, no data provided" # Only for internal use, all other use will return an error if (request.remote_addr != '0.0.0.0') and (request.remote_addr != SERVER_IP): return "INVALID, API for internal use only" dictdata = ImmutableMultiDict(request.form).to_dict(flat='') TOK = dictdata["token"][0] if pp.token_test(TOK) == False: return "INVALID token" # Checks if user wants boinc2docker or adtd-p try: boapp = dictdata['boapp'][0].lower() if (boapp != "boinc2docker") and (boapp != "adtdp"): return "INVALID application" except: return "INVALID, application not provided" file = request.files['file'] # Files must be JSON if file.filename == '': return "INVALID, no file provided" if file.filename.split('.')[-1] != 'json': return "File type invalid, only JSON files are acccepted" # Writes the commands to a random file new_filename = pp.random_file_name() file.save(UPLOAD_FOLDER + new_filename) try: with open(UPLOAD_FOLDER + new_filename, 'r') as ff: command_data = json.load(ff) except: os.remove(UPLOAD_FOLDER + new_filename) return "INVALID, json could not be parsed" # All JSON files must have the following: # Image, Image is custom, Command try: Image = command_data["Image"] Custom = command_data["Custom"] Command = command_data["Command"] except: os.remove(UPLOAD_FOLDER + new_filename) return "INVALID, json lacks at least one field (keys: Image, Custom, Command)" with open(UPLOAD_FOLDER + new_filename, "w") as comfil: comfil.write(Image + "/bin/bash -c ") Invalid_Custom = False # Custom images require more work because we must ensure the results will be back if Custom == "Yes": comfil.write( "\"" + Command + "; mkdir -p /root/shared/results/; mv ./* /root/shared/results\"" ) comfil.write("\n" + str(TOK)) elif Custom == "No": comfil.write("\"cd /data; " + Command + "; python /Mov_Res.py\"") comfil.write("\n" + str(TOK)) else: Invalid_Custom = True if Invalid_Custom: # There is no need to keep the file after if has been deleted os.remove(UPLOAD_FOLDER + new_filename) return "INVALID, custom value can only be Yes/No" # Submits the file for processing if boapp == "boinc2docker": shutil.move(UPLOAD_FOLDER + new_filename, BOINC_FOLDER + new_filename) if boapp == "adtdp": shutil.move(UPLOAD_FOLDER + new_filename, ADTDP_FOLDER + new_filename) return "Commands submitted for processing"
def upload_file(toktok, Username): TOK = toktok if pp.token_test(TOK) == False: return 'Invalid token' if request.method != 'POST': return "Invalid. Submit a text file" file = request.files['file'] # Gathers all the data try: dictdata = ImmutableMultiDict(request.form).to_dict(flat='') app = str(dictdata["app"][0].lower()) try: tags_used = [x.strip() for x in dictdata["topics"][0].split(";") if x.strip() != ""] if tags_used == []: tags_used = "STEM" else: tags_used = ",".join(tags_used) tags_used = tags_used.lower() except Exception as e: print(e) # Error in processing json tags_used = "STEM" except: return "INVALID, not all data has been provided" # If no app is provided, it will assume BOINC try: if (app != "boinc2docker") and (app != "adtdp"): return "INVALID app" except: app = "boinc2docker" # Avoids empty files and non-text files if file.filename == '': return 'No file submitted' if file.filename.split('.')[-1] != 'txt': return "File type invalid, only text files are acccepted" # Randomizes the file name to avoid matches new_filename = pp.random_file_name() file.save(os.path.join(UPLOAD_FOLDER, new_filename)) # Gets the image names # Although uncommon, users can write multiple commands in a single file with open(UPLOAD_FOLDER+'/'+new_filename, 'r') as comfil: for line in comfil: LL = line.replace('\n', '') if len(line.split(' ')) == 1: continue an_image = line.split(' ')[0] a_command = " ".join(line.split(' ')[1:]) if image_is_TACC(an_image): # Gets command command_itself = a_command.replace('/bin/bash -c "cd /data;', "") command_itself = command_itself.replace("mv ./* /root/shared/results\"", "") # Escapes quotes command_itself = sanitize_str_chars(command_itself) Complete_command = "/bin/bash -c \""+command_itself+" mv ./* /root/shared/results\"" # Custom image else: # Gets command command_itself = a_command.replace("/bin/bash -c \"", "") command_itself = a_command.replace("mv ./* /root/shared/results\"", "") # Escapes quotes command_itself = sanitize_str_chars(command_itself) Complete_command = "/bin/bash -c \"mkdir -p /data; cd /data; "+command_itself+" mkdir -p /root/shared/results/; mv ./* /root/shared/results" # Adds job to database mints.add_boinc2docker_job(Username, TOK, tags_used, an_image, Complete_command, "boinc2docker", "cli", "Job submitted") # Removes file os.remove(UPLOAD_FOLDER+'/'+new_filename) return "File submitted for processing\n"
def all_user_files(toktok): if pp.token_test(toktok) == False: return 'Invalid token' resp = requests.get('http://'+os.environ['Reef_IP']+':2000/reef/all_user_files/'+os.environ['Reef_Key']+'/'+toktok) return resp.text