def midas(toktok): if pp.token_test(toktok) == False: return 'Invalid token' if request.method != 'POST': return 'Invalid, no file submitted' file = request.files['file'] try: ALL_USER_DATA = os.listdir( '/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok)) except: return 'User sandbox is not set-up, create a sandbox first' # If no app is provided, it will assume BOINC try: boapp = request.form["app"].lower() if (boapp != "boinc2docker") and (boapp != "adtdp"): return "INVALID app" except: boapp = "boinc2docker" # No user can submit jobs with a full allocation assigned_allocation = float(r.get(toktok).decode('UTF-8')) if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824): return 'User has exceded asssigned allocation. Current available allocation is ' + str( assigned_allocation) + ' GB' if file.filename == '': return 'Invalid, no file uploaded' if ',' in file.filename: return "ERROR: No ',' allowed in filenames" if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename): return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz' new_name = secure_filename(file.filename) file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) try: TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_name) except: return 'ERROR: python cannot open tar file' if not any('README.txt' in str(f) for f in TAR.getmembers()): os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name)) return 'ERROR: tar file does not contain mandatory README.txt' # Creates a new MIDAS directory with a new name while True: new_MID = 'MID_' + pp.random_dir_name() if new_MID not in ALL_USER_DATA: break # Checks the README for all necessary inputs TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID os.makedirs(TAR_PATH) TAR.extractall(TAR_PATH) if not mdr.valid_README(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: README is not valid' if not mdr.valid_OS(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: OS is not accepted' if not mdr.present_input_files(TAR_PATH): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Not all input files for commands are present' if not mdr.valid_language(TAR_PATH + '/README.txt'): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language is not accepted' # Avoids cases where no libraries are needed at all if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and ( mdr.install_libraries(TAR_PATH + '/README.txt') != []): shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok) + '/' + new_MID) return 'ERROR: Language does not support libraries' # Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp} r.set(toktok + ';' + new_MID, boapp) return 'File submitted for processing'
# Goes one by one in the list of submitted files for HJK in to_be_processed: user_tok, dir_midas = HJK FILE_LOCATION = "/root/project/api/sandbox_files/DIR_"+user_tok+'/'+dir_midas # Goes to the file location os.chdir(FILE_LOCATION) #hti: how to install hti_OS = mdr.install_OS('README.txt') hti_langs =[mdr.install_language(y, mdr.valid_OS('README.txt')) for y in mdr.valid_language('README.txt')] hti_setup = mdr.user_guided_setup('README.txt') hti_libs = mdr.install_libraries('README.txt') # Obtains the commands to run ALL_COMS = mdr.present_input_files('.') FINAL_COMMANDS = [] for acom in ALL_COMS: # Other languages FINAL_COMMANDS.append(mdr.execute_command(acom)) # All dockerfiles are named the same way {TOKEN}:{10 char hash} namran = hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:10:] DTAG = (user_tok+':'+namran).lower() # Composes the dockerfile duck = hti_OS+"\n\n\n"+"\n".join(mdr.copy_files_to_image('.')) duck += "\nRUN "+" && ".join(hti_langs) for inst_set in [hti_setup, hti_libs]:
def process_midas_jobs(): try: dictdata = request.get_json() except: return "INVALID, JSON could not be parsed" try: TOK = dictdata["token"] tmp_dir = dictdata["folder_name"] OS = dictdata["operating_system"] prolangs = dictdata["programming_language"] libs = dictdata["library_list"] setfiles = dictdata["setup_filename"] outfiles = dictdata["output_file"] coms = dictdata["command_lines"] except: return "INVALID, json lacks at least one field" if pp.token_test(TOK) == False: shutil.rmtree(TMP) return 'INVALID, invalid token' # MIDAS files are stored in a temporary folder boapp = "boinc2docker" TMP = "/tmp/" + tmp_dir + '/' if 'README.txt' in os.listdir(TMP): return "INVALID, README.txt is a MIDAS reserved file. Please, remove it and try again" for file in os.listdir(TMP): if (file.split(".")[-1] == "tgz") or (".".join( file.split(".")[::-1][:2:][::-1]) == "tar.gz"): # Takes the contents out and deletes the tar try: tar = tarfile.open(TMP + file) tar.extractall(TMP) tar.close() os.remove(TMP + file) continue except: shutil.rmtree(TMP) return "INVALID, cannot open tar file" if (file.split(".")[-1] == "zip"): try: zip_ref = zipfile.ZipFile(TMP + file, 'r') zip_ref.extractall(TMP) zip_ref.close() os.remove(TMP + file) except: shutil.rmtree(TMP) return "INVALID, cannot open zip file" # The application is adtdp if using c++ libraries if libs['cPlusPlus'] != []: boapp = "adtdp" # Creates an acceptable README with open(TMP + "README.txt", 'w') as readme: try: readme.write(verne(dictdata, TMP, boapp)) except Exception as e: shutil.rmtree(TMP) return "INVALID, " + str(e) # Validates if the file can be processed if not mdr.valid_README(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, README in user input' if not mdr.valid_OS(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, OS is not accepted' if not mdr.present_input_files(TMP): shutil.rmtree(TMP) return 'INVALID, Not all input files for commands are present' if not mdr.valid_language(TMP + '/README.txt'): shutil.rmtree(TMP) return 'INVALID, Language is not accepted' # Avoids cases where no libraries are needed at all if (not mdr.install_libraries(TMP + '/README.txt')) and ( mdr.install_libraries(TMP + '/README.txt') != []): shutil.rmtree(TMP) return 'INVALID, Language does not support libraries' # Moves the directory to MIDAS and processes it ALL_USER_DATA = os.listdir('/root/project/api/sandbox_files/DIR_' + str(TOK)) while True: new_MID = 'MID_' + pp.random_dir_name() if new_MID not in ALL_USER_DATA: break MIDAS_PATH = UPLOAD_FOLDER + '/DIR_' + str(TOK) + '/' + new_MID shutil.copytree(TMP, MIDAS_PATH) shutil.rmtree(TMP) # Creates a redis database with syntax {TOKEN}.{MID_DIRECTORY} r.set(TOK + ';' + new_MID, boapp) return 'File submitted for processing'