def midas(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, no file submitted'
file = request.files['file']
try:
ALL_USER_DATA = os.listdir(
'/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok))
except:
return 'User sandbox is not set-up, create a sandbox first'
# If no app is provided, it will assume BOINC
try:
boapp = request.form["app"].lower()
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID app"
except:
boapp = "boinc2docker"
# No user can submit jobs with a full allocation
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824):
return 'User has exceded asssigned allocation. Current available allocation is ' + str(
assigned_allocation) + ' GB'
if file.filename == '':
return 'Invalid, no file uploaded'
if ',' in file.filename:
return "ERROR: No ',' allowed in filenames"
if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename):
return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz'
new_name = secure_filename(file.filename)
file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name))
try:
TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' +
new_name)
except:
return 'ERROR: python cannot open tar file'
if not any('README.txt' in str(f) for f in TAR.getmembers()):
os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok),
new_name))
return 'ERROR: tar file does not contain mandatory README.txt'
# Creates a new MIDAS directory with a new name
while True:
new_MID = 'MID_' + pp.random_dir_name()
if new_MID not in ALL_USER_DATA: break
# Checks the README for all necessary inputs
TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID
os.makedirs(TAR_PATH)
TAR.extractall(TAR_PATH)
if not mdr.valid_README(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: README is not valid'
if not mdr.valid_OS(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: OS is not accepted'
if not mdr.present_input_files(TAR_PATH):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Not all input files for commands are present'
if not mdr.valid_language(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language is not accepted'
# Avoids cases where no libraries are needed at all
if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and (
mdr.install_libraries(TAR_PATH + '/README.txt') != []):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language does not support libraries'
# Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp}
r.set(toktok + ';' + new_MID, boapp)
return 'File submitted for processing'
# Goes one by one in the list of submitted files
for HJK in to_be_processed:
user_tok, dir_midas = HJK
FILE_LOCATION = "/root/project/api/sandbox_files/DIR_"+user_tok+'/'+dir_midas
# Goes to the file location
os.chdir(FILE_LOCATION)
#hti: how to install
hti_OS = mdr.install_OS('README.txt')
hti_langs =[mdr.install_language(y, mdr.valid_OS('README.txt')) for y in mdr.valid_language('README.txt')]
hti_setup = mdr.user_guided_setup('README.txt')
hti_libs = mdr.install_libraries('README.txt')
# Obtains the commands to run
ALL_COMS = mdr.present_input_files('.')
FINAL_COMMANDS = []
for acom in ALL_COMS:
# Other languages
FINAL_COMMANDS.append(mdr.execute_command(acom))
# All dockerfiles are named the same way {TOKEN}:{10 char hash}
namran = hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:10:]
DTAG = (user_tok+':'+namran).lower()
# Composes the dockerfile
duck = hti_OS+"\n\n\n"+"\n".join(mdr.copy_files_to_image('.'))
duck += "\nRUN "+" && ".join(hti_langs)
for inst_set in [hti_setup, hti_libs]:
def process_midas_jobs():
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["token"]
tmp_dir = dictdata["folder_name"]
OS = dictdata["operating_system"]
prolangs = dictdata["programming_language"]
libs = dictdata["library_list"]
setfiles = dictdata["setup_filename"]
outfiles = dictdata["output_file"]
coms = dictdata["command_lines"]
except:
return "INVALID, json lacks at least one field"
if pp.token_test(TOK) == False:
shutil.rmtree(TMP)
return 'INVALID, invalid token'
# MIDAS files are stored in a temporary folder
boapp = "boinc2docker"
TMP = "/tmp/" + tmp_dir + '/'
if 'README.txt' in os.listdir(TMP):
return "INVALID, README.txt is a MIDAS reserved file. Please, remove it and try again"
for file in os.listdir(TMP):
if (file.split(".")[-1] == "tgz") or (".".join(
file.split(".")[::-1][:2:][::-1]) == "tar.gz"):
# Takes the contents out and deletes the tar
try:
tar = tarfile.open(TMP + file)
tar.extractall(TMP)
tar.close()
os.remove(TMP + file)
continue
except:
shutil.rmtree(TMP)
return "INVALID, cannot open tar file"
if (file.split(".")[-1] == "zip"):
try:
zip_ref = zipfile.ZipFile(TMP + file, 'r')
zip_ref.extractall(TMP)
zip_ref.close()
os.remove(TMP + file)
except:
shutil.rmtree(TMP)
return "INVALID, cannot open zip file"
# The application is adtdp if using c++ libraries
if libs['cPlusPlus'] != []:
boapp = "adtdp"
# Creates an acceptable README
with open(TMP + "README.txt", 'w') as readme:
try:
readme.write(verne(dictdata, TMP, boapp))
except Exception as e:
shutil.rmtree(TMP)
return "INVALID, " + str(e)
# Validates if the file can be processed
if not mdr.valid_README(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, README in user input'
if not mdr.valid_OS(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, OS is not accepted'
if not mdr.present_input_files(TMP):
shutil.rmtree(TMP)
return 'INVALID, Not all input files for commands are present'
if not mdr.valid_language(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, Language is not accepted'
# Avoids cases where no libraries are needed at all
if (not mdr.install_libraries(TMP + '/README.txt')) and (
mdr.install_libraries(TMP + '/README.txt') != []):
shutil.rmtree(TMP)
return 'INVALID, Language does not support libraries'
# Moves the directory to MIDAS and processes it
ALL_USER_DATA = os.listdir('/root/project/api/sandbox_files/DIR_' +
str(TOK))
while True:
new_MID = 'MID_' + pp.random_dir_name()
if new_MID not in ALL_USER_DATA:
break
MIDAS_PATH = UPLOAD_FOLDER + '/DIR_' + str(TOK) + '/' + new_MID
shutil.copytree(TMP, MIDAS_PATH)
shutil.rmtree(TMP)
# Creates a redis database with syntax {TOKEN}.{MID_DIRECTORY}
r.set(TOK + ';' + new_MID, boapp)
return 'File submitted for processing'