def upload_file(toktok, Username): TOK = toktok if pp.token_test(TOK) == False: return 'Invalid token' if request.method != 'POST': return "Invalid. Submit a text file" file = request.files['file'] # Gathers all the data try: dictdata = ImmutableMultiDict(request.form).to_dict(flat='') app = str(dictdata["app"][0].lower()) try: tags_used = [x.strip() for x in dictdata["topics"][0].split(";") if x.strip() != ""] if tags_used == []: tags_used = "STEM" else: tags_used = ",".join(tags_used) tags_used = tags_used.lower() except Exception as e: print(e) # Error in processing json tags_used = "STEM" except: return "INVALID, not all data has been provided" # If no app is provided, it will assume BOINC try: if (app != "boinc2docker") and (app != "adtdp"): return "INVALID app" except: app = "boinc2docker" # Avoids empty files and non-text files if file.filename == '': return 'No file submitted' if file.filename.split('.')[-1] != 'txt': return "File type invalid, only text files are acccepted" # Randomizes the file name to avoid matches new_filename = pp.random_file_name() file.save(os.path.join(UPLOAD_FOLDER, new_filename)) # Gets the image names # Although uncommon, users can write multiple commands in a single file with open(UPLOAD_FOLDER+'/'+new_filename, 'r') as comfil: for line in comfil: LL = line.replace('\n', '') if len(line.split(' ')) == 1: continue an_image = line.split(' ')[0] a_command = " ".join(line.split(' ')[1:]) if image_is_TACC(an_image): # Gets command command_itself = a_command.replace('/bin/bash -c "cd /data;', "") command_itself = command_itself.replace("mv ./* /root/shared/results\"", "") # Escapes quotes command_itself = sanitize_str_chars(command_itself) Complete_command = "/bin/bash -c \""+command_itself+" mv ./* /root/shared/results\"" # Custom image else: # Gets command command_itself = a_command.replace("/bin/bash -c \"", "") command_itself = a_command.replace("mv ./* /root/shared/results\"", "") # Escapes quotes command_itself = sanitize_str_chars(command_itself) Complete_command = "/bin/bash -c \"mkdir -p /data; cd /data; "+command_itself+" mkdir -p /root/shared/results/; mv ./* /root/shared/results" # Adds job to database mints.add_boinc2docker_job(Username, TOK, tags_used, an_image, Complete_command, "boinc2docker", "cli", "Job submitted") # Removes file os.remove(UPLOAD_FOLDER+'/'+new_filename) return "File submitted for processing\n"
def process_web_jobs(): if request.method != 'POST': return "INVALID, no data provided" try: dictdata = request.get_json() except: return "INVALID, JSON could not be parsed" try: TOK = dictdata["Token"] Reefil = dictdata["Files"] Image = dictdata["Image"] Custom = dictdata["Custom"] Command = dictdata["Command"] Username = dictdata["Username"] if "priority" not in dictdata.keys(): PRIORITY = "Middle" else: PRIORITY = dictdata["priority"] except: return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command, Username)" if pp.token_test(TOK) == False: return "INVALID token" # Checks the list of Commands to ensure their correct execution Command = ';'.join([C for C in Command.split(';') if (C != '') and (C.count(C[0]) != len(C)) ]) +';' boapp = "boinc2docker" if Image == "carlosred/gpu:cuda": boapp = "volcon" if (Custom != "Yes" ) and (Custom != "No"): return abort(422) # Incorrect API if not image_is_TACC(Image): Custom = "Yes" # Gets the tags try: tags_used = [x.strip() for x in dictdata["topics"].split(";") if x.strip() != ""] if tags_used == []: tags_used = "STEM" else: tags_used = ",".join(tags_used) tags_used = tags_used.lower() except Exception as e: print(e) # Error in processing json tags_used = "STEM" else: # Default tag: STEM tags_used = "STEM" if boapp == "boinc2docker": # Writes the commands to a random file new_filename = pp.random_file_name() Complete_command = "" # Custom images require more work because it must be ensured the results will be back if Custom == "Yes": # Creates a new working directory Complete_command += "mkdir -p /data; cd /data; " # Adds the files for FF in Reefil: # Skips unnecessary files if FF == "": break Complete_command += get_reef_file(Image, TOK, FF)+" " Complete_command += Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results" elif Custom == "No": # Adds the files for FF in Reefil: # Skips unnecessary files if FF == "": break Complete_command += get_reef_file(Image, TOK, FF)+" " Complete_command += Command +" mv ./* /root/shared/results" # Replaces certain characters Complete_command = " /bin/bash -c \""+sanitize_str_chars(Complete_command)+"\"" mints.add_boinc2docker_job(Username, TOK, tags_used, Image, Complete_command, boapp, "web", "Job submitted") # Old way # shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename) if boapp == "volcon": # Only CUDA requires a GPU # Custom images are also assumed to not require a GPU TODO TODO TODO TODO if Image == "carlosred/gpu:cuda": GPU = 1 else: GPU = 0 VolCon_ID = uuid.uuid4().hex COMMANDS = "" if Custom == "Yes": TACC = 0 COMMANDS += "mkdir -p /data; cd /data; " for FF in Reefil: if FF == '': break COMMANDS += get_reef_file(Image, TOK, FF)+" " # Splits the commands and ensures that they are run in /data newcoms = ";".join(["cd /data && "+x for x in Command.split(";")]) COMMANDS += newcoms+" mkdir -p /root/shared/results/; mv /data/* /root/shared/results" else: TACC = 1 for FF in Reefil: if FF == '': break COMMANDS += get_reef_file(Image, TOK, FF)+" " COMMANDS += ";".join([extra_image_commands(Image) +z for z in Command.split(";")])+" mv ./* /root/shared/results" COMMANDS = " /bin/bash -c \""+COMMANDS+"\"" job_info = {"Image":Image, "Command":COMMANDS, "TACC":TACC, "GPU":GPU, "VolCon_ID":VolCon_ID, "public":1} # Updates the job in the database mints.add_job(TOK, Image, COMMANDS, GPU, VolCon_ID, PRIORITY, 1, tags_used, Username, "web") # Pushes job information to mirrors mirror.upload_job_to_mirror(job_info) return "Commands submitted for processing"