def upload_file(toktok):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Adds the token at the end of the file
with open(UPLOAD_FOLDER+'/'+new_filename, 'a') as nod:
nod.write('\n'+str(TOK))
shutil.move(UPLOAD_FOLDER+'/'+new_filename, FINAL_FOLDER+'/'+new_filename)
return "File submitted for processing"
def upload_file(toktok):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# If no app is provided, it will assume BOINC
try:
app = request.form["app"].lower()
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Adds the token at the end of the file
with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod:
nod.write('\n' + str(TOK))
if app == "boinc2docker":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
FINAL_FOLDER + '/' + new_filename)
if app == "adtdp":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
ADTDP_FOLDER + '/' + new_filename)
return "File submitted for processing"
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
# Only for internal use, all other use will return an error
if (request.remote_addr != '0.0.0.0') and (request.remote_addr !=
SERVER_IP):
return "INVALID, API for internal use only"
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
TOK = dictdata["token"][0]
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks if user wants boinc2docker or adtd-p
try:
boapp = dictdata['boapp'][0].lower()
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID application"
except:
return "INVALID, application not provided"
file = request.files['file']
# Files must be JSON
if file.filename == '':
return "INVALID, no file provided"
if file.filename.split('.')[-1] != 'json':
return "File type invalid, only JSON files are acccepted"
# Writes the commands to a random file
new_filename = pp.random_file_name()
file.save(UPLOAD_FOLDER + new_filename)
try:
with open(UPLOAD_FOLDER + new_filename, 'r') as ff:
command_data = json.load(ff)
except:
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, json could not be parsed"
# All JSON files must have the following:
# Image, Image is custom, Command
try:
Image = command_data["Image"]
Custom = command_data["Custom"]
Command = command_data["Command"]
except:
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, json lacks at least one field (keys: Image, Custom, Command)"
with open(UPLOAD_FOLDER + new_filename, "w") as comfil:
comfil.write(Image + "/bin/bash -c ")
Invalid_Custom = False
# Custom images require more work because we must ensure the results will be back
if Custom == "Yes":
comfil.write(
"\"" + Command +
"; mkdir -p /root/shared/results/; mv ./* /root/shared/results\""
)
comfil.write("\n" + str(TOK))
elif Custom == "No":
comfil.write("\"cd /data; " + Command + "; python /Mov_Res.py\"")
comfil.write("\n" + str(TOK))
else:
Invalid_Custom = True
if Invalid_Custom:
# There is no need to keep the file after if has been deleted
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, custom value can only be Yes/No"
# Submits the file for processing
if boapp == "boinc2docker":
shutil.move(UPLOAD_FOLDER + new_filename, BOINC_FOLDER + new_filename)
if boapp == "adtdp":
shutil.move(UPLOAD_FOLDER + new_filename, ADTDP_FOLDER + new_filename)
return "Commands submitted for processing"
def upload_file(toktok, Username):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# Gathers all the data
try:
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
app = str(dictdata["app"][0].lower())
try:
tags_used = [x.strip() for x in dictdata["topics"][0].split(";") if x.strip() != ""]
if tags_used == []:
tags_used = "STEM"
else:
tags_used = ",".join(tags_used)
tags_used = tags_used.lower()
except Exception as e:
print(e)
# Error in processing json
tags_used = "STEM"
except:
return "INVALID, not all data has been provided"
# If no app is provided, it will assume BOINC
try:
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Gets the image names
# Although uncommon, users can write multiple commands in a single file
with open(UPLOAD_FOLDER+'/'+new_filename, 'r') as comfil:
for line in comfil:
LL = line.replace('\n', '')
if len(line.split(' ')) == 1:
continue
an_image = line.split(' ')[0]
a_command = " ".join(line.split(' ')[1:])
if image_is_TACC(an_image):
# Gets command
command_itself = a_command.replace('/bin/bash -c "cd /data;', "")
command_itself = command_itself.replace("mv ./* /root/shared/results\"", "")
# Escapes quotes
command_itself = sanitize_str_chars(command_itself)
Complete_command = "/bin/bash -c \""+command_itself+" mv ./* /root/shared/results\""
# Custom image
else:
# Gets command
command_itself = a_command.replace("/bin/bash -c \"", "")
command_itself = a_command.replace("mv ./* /root/shared/results\"", "")
# Escapes quotes
command_itself = sanitize_str_chars(command_itself)
Complete_command = "/bin/bash -c \"mkdir -p /data; cd /data; "+command_itself+" mkdir -p /root/shared/results/; mv ./* /root/shared/results"
# Adds job to database
mints.add_boinc2docker_job(Username, TOK, tags_used, an_image, Complete_command, "boinc2docker", "cli", "Job submitted")
# Removes file
os.remove(UPLOAD_FOLDER+'/'+new_filename)
return "File submitted for processing\n"
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
boapp = dictdata['Boapp'].lower()
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
tprov = json.loads(dictdata["topics"])
# TACC images have already assigned topics
if Image in cus.at.TACCIM.keys():
tprov = cus.at.TACCIM[Image]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks if user wants boinc2docker or adtd-p
try:
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID application"
except:
return "INVALID, application not provided"
if (Custom != "Yes") and (Custom != "No"):
return "INVALID, Custom value can only be [Yes/No]"
if ("Custom" == "No") and (not cus.image_is_TACC(Image)):
return "INVALID, Image \'"+Image+"\' is not provided by TACC"
# Writes the commands to a random file
new_filename = pp.random_file_name()
with open(UPLOAD_FOLDER+new_filename, "w") as comfil:
comfil.write(Image + " /bin/bash -c ")
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
# Creates a new working directory
comfil.write("\"mkdir -p /data; cd /data; ")
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results\"")
comfil.write("\n"+str(TOK))
elif Custom == "No":
comfil.write("\"")
comfil.write(extra_image_commands(Image))
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" python /Mov_Res.py\"")
comfil.write("\n"+str(TOK))
# Submits the file for processing
if boapp == "boinc2docker":
toproc(Image, tprov)
shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "adtdp":
shutil.move(UPLOAD_FOLDER+new_filename, ADTDP_FOLDER+new_filename)
return "Commands submitted for processing"
def upload_file(toktok):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# Gathers all the data
try:
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
app = str(dictdata["app"][0].lower())
# Gets the list of topics/subtopics
Tops = [j for j in dictdata.keys() if j != "app"]
# Each topic will be defined by Topic: subtopics
# Subtopics are separated by commas
TST = {} # Topics structure
for one_top in Tops:
TST[one_top] = [
zz for zz in dictdata[one_top][0].upper().split(',')
if zz != ''
]
except:
return "INVALID, not all data has been provided"
# If no app is provided, it will assume BOINC
try:
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Gets the image names
# Although uncommon, users can write multiple commands in a single file
Im_Names = []
with open(UPLOAD_FOLDER + '/' + new_filename, 'r') as comfil:
for line in comfil:
LL = line.replace('\n', '')
if len(line.split(' ')) == 1:
continue
Im_Names.append(line.split(' ')[0])
# Adds the token at the end of the file
with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod:
nod.write('\n' + str(TOK))
AA = ''
if app == "boinc2docker":
# Adds the topic to database
for an_image in Im_Names:
AA += cus.complete_tag_work(an_image, TST, jobsub=len(Im_Names))
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
FINAL_FOLDER + '/' + new_filename)
if app == "adtdp":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
ADTDP_FOLDER + '/' + new_filename)
return "File submitted for processing\n" + AA
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
Username = dictdata["Username"]
if "priority" not in dictdata.keys():
PRIORITY = "Middle"
else:
PRIORITY = dictdata["priority"]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command, Username)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks the list of Commands to ensure their correct execution
Command = ';'.join([C for C in Command.split(';') if (C != '') and (C.count(C[0]) != len(C)) ]) +';'
boapp = "boinc2docker"
if Image == "carlosred/gpu:cuda":
boapp = "volcon"
if (Custom != "Yes" ) and (Custom != "No"):
return abort(422) # Incorrect API
if not image_is_TACC(Image):
Custom = "Yes"
# Gets the tags
try:
tags_used = [x.strip() for x in dictdata["topics"].split(";") if x.strip() != ""]
if tags_used == []:
tags_used = "STEM"
else:
tags_used = ",".join(tags_used)
tags_used = tags_used.lower()
except Exception as e:
print(e)
# Error in processing json
tags_used = "STEM"
else:
# Default tag: STEM
tags_used = "STEM"
if boapp == "boinc2docker":
# Writes the commands to a random file
new_filename = pp.random_file_name()
Complete_command = ""
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
# Creates a new working directory
Complete_command += "mkdir -p /data; cd /data; "
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results"
elif Custom == "No":
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command +" mv ./* /root/shared/results"
# Replaces certain characters
Complete_command = " /bin/bash -c \""+sanitize_str_chars(Complete_command)+"\""
mints.add_boinc2docker_job(Username, TOK, tags_used, Image, Complete_command, boapp, "web", "Job submitted")
# Old way
# shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "volcon":
# Only CUDA requires a GPU
# Custom images are also assumed to not require a GPU TODO TODO TODO TODO
if Image == "carlosred/gpu:cuda":
GPU = 1
else:
GPU = 0
VolCon_ID = uuid.uuid4().hex
COMMANDS = ""
if Custom == "Yes":
TACC = 0
COMMANDS += "mkdir -p /data; cd /data; "
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
# Splits the commands and ensures that they are run in /data
newcoms = ";".join(["cd /data && "+x for x in Command.split(";")])
COMMANDS += newcoms+" mkdir -p /root/shared/results/; mv /data/* /root/shared/results"
else:
TACC = 1
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
COMMANDS += ";".join([extra_image_commands(Image) +z for z in Command.split(";")])+" mv ./* /root/shared/results"
COMMANDS = " /bin/bash -c \""+COMMANDS+"\""
job_info = {"Image":Image, "Command":COMMANDS, "TACC":TACC, "GPU":GPU, "VolCon_ID":VolCon_ID, "public":1}
# Updates the job in the database
mints.add_job(TOK, Image, COMMANDS, GPU, VolCon_ID, PRIORITY, 1, tags_used, Username, "web")
# Pushes job information to mirrors
mirror.upload_job_to_mirror(job_info)
return "Commands submitted for processing"
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
# Only for internal use, all other use will return an error
if (request.remote_addr != '0.0.0.0') and (request.remote_addr != SERVER_IP):
return "INVALID, API for internal use only"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
boapp = dictdata['Boapp'].lower()
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks if user wants boinc2docker or adtd-p
try:
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID application"
except:
return "INVALID, application not provided"
if (Custom != "Yes") and (Custom != "No"):
return "INVALID, Custom value can only be [Yes/No]"
# Writes the commands to a random file
new_filename = pp.random_file_name()
with open(UPLOAD_FOLDER+new_filename, "w") as comfil:
comfil.write(Image + " /bin/bash -c ")
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
comfil.write("\"")
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results\"")
comfil.write("\n"+str(TOK))
elif Custom == "No":
comfil.write("\"cd /data; ")
comfil.write(extra_image_commands(Image))
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" python /Mov_Res.py\"")
comfil.write("\n"+str(TOK))
# Submits the file for processing
if boapp == "boinc2docker":
shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "adtdp":
shutil.move(UPLOAD_FOLDER+new_filename, ADTDP_FOLDER+new_filename)
return "Commands submitted for processing"