def new_sandbox():
if request.method != 'POST':
return 'Invalid, provide a token'
TOK = request.form['token']
if pp.token_test(TOK) == False:
return 'Invalid token'
resp = requests.get('http://'+os.environ['Reef_IP']+':2002/reef/create_user/'+TOK+'/'+os.environ['Reef_Key'])
return resp.text
def reef_results_all(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if str('DIR_' +
toktok) not in os.listdir('/root/project/api/sandbox_files'):
return 'User directory does not exist'
USER_DIR = '/root/project/api/sandbox_files/DIR_' + str(
toktok) + '/___RESULTS'
# Returns the results (space-separated)
return ' '.join(os.listdir(USER_DIR))
def obtain_file(FIL, toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if str('DIR_'+toktok) not in os.listdir('/root/project/api/sandbox_files'):
return 'User directory does not exist'
USER_DIR = '/root/project/api/sandbox_files/DIR_'+str(toktok)+'/'
if str(FIL) not in os.listdir(USER_DIR):
return 'File not available'
return send_file(USER_DIR+str(FIL))
def user_images(toktok):
if not pp.token_test(toktok):
return 'Invalid token'
user_images = {}
for IMAGE in client.images.list():
try:
NAME = IMAGE.attrs['RepoTags'][0]
except:
continue
if NAME.split(':')[0] == toktok.lower():
user_images[NAME] = str(IMAGE.attrs['Size'] / (10**9)) + " GB"
return jsonify(user_images)
def all_user_files(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
# Accounts for users without a sandbox yet
try:
AAA = []
for afil in os.listdir('/root/project/api/sandbox_files/DIR_'+str(toktok)):
AAA.append(afil)
return ','.join(AAA)
except:
return 'Sandbox not set-up, create a sandbox first'
def tacc_jobs():
# Ensures that there is an appropriate json request
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
if pp.token_test(proposal["token"]) == False:
return 'Invalid token'
# Checks the required fields
req_fields = ["token", "image", "commands", "priority"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: " + ",".join(
[str(a) for a in req_check])
[TOKEN, IMAGE, COMMANDS, PRIORITY] = [
proposal["token"], proposal["image"], proposal["commands"],
proposal["priority"]
]
VolCon_ID = uuid.uuid4().hex
if "gpu" in IMAGE:
GPU = 1
else:
GPU = 0
try:
mints.add_job(TOKEN, IMAGE, COMMANDS, GPU, VolCon_ID, PRIORITY)
except:
return "INVALID: Could not connect to MySQL database"
# TACC: Image is a TACC image
job_info = {
"Image": IMAGE,
"Command": COMMANDS,
"TACC": 1,
"GPU": GPU,
"VolCon_ID": VolCon_ID,
"public": 1
}
mirror.upload_job_to_mirror(job_info)
return "Successfully submitted job"
def delete_user_file(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, provide a file to be deleted'
try:
FILE = request.form['del']
if FILE == '':
return 'No file provided'
except:
return "Cannot parse request, 'del' entry is not present"
resp = requests.get('http://'+os.environ['Reef_IP']+':2000/reef/delete_file/'+os.environ['Reef_Key']+'/'+toktok+'/'+FILE)
return resp.text
def reef_allocation_status(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
used_space = pp.user_sandbox_size(str(toktok)) / 1073741824
assigned_allocation = r.get(toktok).decode('UTF-8')
all_info = {
'Max. allocation':
assigned_allocation + ' GB',
'Used space':
str(used_space) + ' GB',
'Space available left':
str((1 - used_space / float(assigned_allocation)) * 100) +
'% allocation available'
}
return jsonify(all_info)
def simple_allocation_check():
if request.method != 'POST':
return "INVALID, no data provided"
if request.form['token'] == '':
return "INVALID, no token provided"
toktok = request.form["token"]
if pp.token_test(toktok) == False:
return 'INVALID token'
assigned_allocation = float(r_alloc.get(toktok).decode('UTF-8'))
if assigned_allocation > 0:
return 'y'
return 'n'
def new_sandbox():
if request.method != 'POST':
return 'Invalid, provide a token'
TOK = request.form['token']
if pp.token_test(TOK) == False:
return 'Invalid token'
# Finds if the directory has already been created or not
for sandir in os.listdir('/root/project/api/sandbox_files'):
if TOK == sandir[4::]:
return 'Sandbox already available'
else:
os.makedirs('/root/project/api/sandbox_files/DIR_' + str(TOK))
os.makedirs('/root/project/api/sandbox_files/DIR_' + str(TOK) +
'/___RESULTS')
return 'reef cloud storage now available'
def references(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
allref = {
'OS': {
'Ubuntu 16.04': 'Ubuntu_16.04'
},
'Languages': {
'python2': 'Not supported',
'python3': 'python, python3',
'Go': 'Go',
'R': 'R',
'Fortran': 'Fortran90',
'C': 'C',
'C++': 'C++'
}
}
return jsonify(allref)
def delete_user_file(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, provide a file to be deleted'
# Accounts for missing directories
if str('DIR_'+toktok) not in os.listdir('/root/project/api/sandbox_files'):
return 'User directory does not exist'
try:
FILE = request.form['del']
if FILE == '':
return 'No file provided'
os.remove('/root/project/api/sandbox_files/DIR_'+str(toktok)+'/'+str(FILE))
return 'File succesfully deleted from reef storage'
except:
return 'File is not present in Reef'
def reef_upload(toktok):
if pp.token_test(toktok) == False:
return 'INVALID token'
if request.method != 'POST':
return 'INVALID, no file submitted'
file = request.files['file']
# Avoids empty filenames and those with commas
if file.filename == '':
return 'INVALID, no file uploaded'
if ',' in file.filename:
return "INVALID, no ',' allowed in filenames"
resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok,
data={"filename":secure_filename(file.filename)}, files={"file": file.read()})
return resp.content
def delete_midas_dir(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, provide a file to be deleted'
# Accounts for missing directories
if str('DIR_' + toktok) not in os.listdir(
'/home/boincadm/project/api/sandbox_files'):
return 'User directory does not exist'
try:
MIDIR = request.form['del']
if MIDIR == '':
return 'No file provided'
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + str(MIDIR))
return 'Midas directory deleted'
except:
return 'MIDAS directory does not exist'
def allocation_status():
if request.method != 'POST':
return "INVALID, no data provided"
if request.form['token'] == '':
return "INVALID, no token provided"
toktok = request.form["token"]
if pp.token_test(toktok) == False:
return 'Invalid token'
used_space = pp.user_sandbox_size(str(toktok))/1073741824
assigned_allocation = r_alloc.get(toktok).decode('UTF-8')
all_info = {'Max. allocation': assigned_allocation+' GB',
'Used space': str(used_space)+' GB',
'Space available left': str((1 - used_space/float(assigned_allocation))*100)+'% allocation available'}
return jsonify(all_info)
def upload_file(toktok):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# If no app is provided, it will assume BOINC
try:
app = request.form["app"].lower()
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Adds the token at the end of the file
with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod:
nod.write('\n' + str(TOK))
if app == "boinc2docker":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
FINAL_FOLDER + '/' + new_filename)
if app == "adtdp":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
ADTDP_FOLDER + '/' + new_filename)
return "File submitted for processing"
def add_username(username, email, toktok, org_key):
org_key = hashlib.sha256(org_key.encode('UTF-8')).hexdigest()[:24:]
all_org_keys = [
r_org.hmget(x.decode('UTF-8'), 'Organization Token')[0].decode('UTF-8')
for x in r_org.keys()
]
if org_key not in all_org_keys:
return 'Organization key invalid, access denied'
if pp.token_test(toktok) == False:
return "INVALID token"
for y in r_org.keys():
if org_key == r_org.hmget(y, 'Organization Token')[0].decode('UTF-8'):
ORG = y.decode('UTF-8')
break
# If the user is already registered it does nothing
for qq in range(0, r.llen(ORG)):
if username == r.lindex(ORG, qq).decode("UTF-8"):
break
else:
# Adds a new row for the user
r.rpush(ORG, username)
# Creates a new hash
r.hmset(username, {email: toktok})
return "Added new username to the database"
# Checks if the user has already set this email
for zz in r.hkeys(username):
if zz.decode('UTF-8') == email:
return "User email already in database"
# Adds another email to the user
r.hset(username, email, toktok)
return "Added another user email"
def delete_image(toktok):
if not pp.token_test(toktok):
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, provide an image to be deleted'
UTOK = str(toktok).lower()
DATA = str(request.form['del'])
IMTAG = DATA
if ':' not in DATA:
IMTAG = UTOK.lower() + ':' + DATA
try:
r.incrbyfloat(toktok,
float(client.images.get(IMTAG).attrs['Size']) / (10**9))
client.images.remove(image=IMTAG, force=True)
return 'User image has been deleted. User allocation is now ' + r.get(
toktok).decode('UTF-8') + " GB"
except:
return 'ERROR: Image does not exist or is not owned by user'
def user_data(toktok):
if pp.token_test(toktok) == False:
return 'INVALID token'
U_alloc = r_alloc.get(toktok).decode("UTF-8")
# Finds all the data
totjobs = r_data.llen("Token")
U_data = []
for qq in range(0, totjobs):
if r_data.lindex("Token", qq).decode("UTF-8") == toktok:
# [[Image, Command, Date (Sub), Date (Run), Notified], ..]
U_data.append([])
curdat = {}
imnam = r_data.lindex("Image", qq).decode("UTF-8")
if "carlosred/" != imnam[:10:]:
imnam += " (CUSTOM)"
curdat["Image"] = imnam
curdat["Command"] = r_data.lindex("Command", qq).decode("UTF-8")
curdat["Date (Sub)"] = transform_to_iso(
r_data.lindex("Date (Sub)", qq).decode("UTF-8"))
curdat["Date (Run)"] = transform_to_iso(
r_data.lindex("Date (Run)", qq).decode("UTF-8"))
curdat["Notified"] = transform_to_iso(
r_data.lindex("Notified", qq).decode("UTF-8"))
U_data[-1].append(curdat)
# Appends the VolCon jobs
vc_info = VolCon_info(toktok)
U_data = U_data + vc_info
# Sorts them by received date
U_data = sorted(U_data, key=lambda x: x[0]["Date (Sub)"])
return jsonify({"allocation": U_alloc, "job data": U_data})
def reef_upload(toktok):
if pp.token_test(toktok) == False:
return 'INVALID token'
if request.method != 'POST':
return 'INVALID, no file submitted'
file = request.files['file']
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation*1073741824):
return 'INVALID, User has exceded asssigned allocation. Max. available allocation is '+str(assigned_allocation)+' GB'
# Avoids empty filenames and those with commas
if file.filename == '':
return 'INVALID, no file uploaded'
if ',' in file.filename:
return "INVALID, no ',' allowed in filenames"
resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok,
data={"filename":secure_filename(file.filename)}, files={"file": file.read()})
return resp.content
def reef_results_all(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
resp = requests.get('http://'+os.environ['Reef_IP']+':2001/reef/results_all/'+os.environ['Reef_Key']+'/'+toktok)
return resp.text
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
Username = dictdata["Username"]
if "priority" not in dictdata.keys():
PRIORITY = "Middle"
else:
PRIORITY = dictdata["priority"]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command, Username)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks the list of Commands to ensure their correct execution
Command = ';'.join([C for C in Command.split(';') if (C != '') and (C.count(C[0]) != len(C)) ]) +';'
boapp = "boinc2docker"
if Image == "carlosred/gpu:cuda":
boapp = "volcon"
if (Custom != "Yes" ) and (Custom != "No"):
return abort(422) # Incorrect API
if not image_is_TACC(Image):
Custom = "Yes"
# Gets the tags
try:
tags_used = [x.strip() for x in dictdata["topics"].split(";") if x.strip() != ""]
if tags_used == []:
tags_used = "STEM"
else:
tags_used = ",".join(tags_used)
tags_used = tags_used.lower()
except Exception as e:
print(e)
# Error in processing json
tags_used = "STEM"
else:
# Default tag: STEM
tags_used = "STEM"
if boapp == "boinc2docker":
# Writes the commands to a random file
new_filename = pp.random_file_name()
Complete_command = ""
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
# Creates a new working directory
Complete_command += "mkdir -p /data; cd /data; "
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results"
elif Custom == "No":
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command +" mv ./* /root/shared/results"
# Replaces certain characters
Complete_command = " /bin/bash -c \""+sanitize_str_chars(Complete_command)+"\""
mints.add_boinc2docker_job(Username, TOK, tags_used, Image, Complete_command, boapp, "web", "Job submitted")
# Old way
# shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "volcon":
# Only CUDA requires a GPU
# Custom images are also assumed to not require a GPU TODO TODO TODO TODO
if Image == "carlosred/gpu:cuda":
GPU = 1
else:
GPU = 0
VolCon_ID = uuid.uuid4().hex
COMMANDS = ""
if Custom == "Yes":
TACC = 0
COMMANDS += "mkdir -p /data; cd /data; "
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
# Splits the commands and ensures that they are run in /data
newcoms = ";".join(["cd /data && "+x for x in Command.split(";")])
COMMANDS += newcoms+" mkdir -p /root/shared/results/; mv /data/* /root/shared/results"
else:
TACC = 1
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
COMMANDS += ";".join([extra_image_commands(Image) +z for z in Command.split(";")])+" mv ./* /root/shared/results"
COMMANDS = " /bin/bash -c \""+COMMANDS+"\""
job_info = {"Image":Image, "Command":COMMANDS, "TACC":TACC, "GPU":GPU, "VolCon_ID":VolCon_ID, "public":1}
# Updates the job in the database
mints.add_job(TOK, Image, COMMANDS, GPU, VolCon_ID, PRIORITY, 1, tags_used, Username, "web")
# Pushes job information to mirrors
mirror.upload_job_to_mirror(job_info)
return "Commands submitted for processing"
def upload_file(toktok):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# Gathers all the data
try:
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
app = str(dictdata["app"][0].lower())
# Gets the list of topics/subtopics
Tops = [j for j in dictdata.keys() if j != "app"]
# Each topic will be defined by Topic: subtopics
# Subtopics are separated by commas
TST = {} # Topics structure
for one_top in Tops:
TST[one_top] = [
zz for zz in dictdata[one_top][0].upper().split(',')
if zz != ''
]
except:
return "INVALID, not all data has been provided"
# If no app is provided, it will assume BOINC
try:
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Gets the image names
# Although uncommon, users can write multiple commands in a single file
Im_Names = []
with open(UPLOAD_FOLDER + '/' + new_filename, 'r') as comfil:
for line in comfil:
LL = line.replace('\n', '')
if len(line.split(' ')) == 1:
continue
Im_Names.append(line.split(' ')[0])
# Adds the token at the end of the file
with open(UPLOAD_FOLDER + '/' + new_filename, 'a') as nod:
nod.write('\n' + str(TOK))
AA = ''
if app == "boinc2docker":
# Adds the topic to database
for an_image in Im_Names:
AA += cus.complete_tag_work(an_image, TST, jobsub=len(Im_Names))
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
FINAL_FOLDER + '/' + new_filename)
if app == "adtdp":
shutil.move(UPLOAD_FOLDER + '/' + new_filename,
ADTDP_FOLDER + '/' + new_filename)
return "File submitted for processing\n" + AA
def midas(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, no file submitted'
file = request.files['file']
try:
ALL_USER_DATA = os.listdir(
'/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok))
except:
return 'User sandbox is not set-up, create a sandbox first'
# If no app is provided, it will assume BOINC
try:
boapp = request.form["app"].lower()
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID app"
except:
boapp = "boinc2docker"
# No user can submit jobs with a full allocation
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824):
return 'User has exceded asssigned allocation. Current available allocation is ' + str(
assigned_allocation) + ' GB'
if file.filename == '':
return 'Invalid, no file uploaded'
if ',' in file.filename:
return "ERROR: No ',' allowed in filenames"
if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename):
return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz'
new_name = secure_filename(file.filename)
file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name))
try:
TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' +
new_name)
except:
return 'ERROR: python cannot open tar file'
if not any('README.txt' in str(f) for f in TAR.getmembers()):
os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok),
new_name))
return 'ERROR: tar file does not contain mandatory README.txt'
# Creates a new MIDAS directory with a new name
while True:
new_MID = 'MID_' + pp.random_dir_name()
if new_MID not in ALL_USER_DATA: break
# Checks the README for all necessary inputs
TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID
os.makedirs(TAR_PATH)
TAR.extractall(TAR_PATH)
if not mdr.valid_README(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: README is not valid'
if not mdr.valid_OS(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: OS is not accepted'
if not mdr.present_input_files(TAR_PATH):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Not all input files for commands are present'
if not mdr.valid_language(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language is not accepted'
# Avoids cases where no libraries are needed at all
if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and (
mdr.install_libraries(TAR_PATH + '/README.txt') != []):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language does not support libraries'
# Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp}
r.set(toktok + ';' + new_MID, boapp)
return 'File submitted for processing'
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
boapp = dictdata['Boapp'].lower()
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
tprov = json.loads(dictdata["topics"])
# TACC images have already assigned topics
if Image in cus.at.TACCIM.keys():
tprov = cus.at.TACCIM[Image]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks if user wants boinc2docker or adtd-p
try:
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID application"
except:
return "INVALID, application not provided"
if (Custom != "Yes") and (Custom != "No"):
return "INVALID, Custom value can only be [Yes/No]"
if ("Custom" == "No") and (not cus.image_is_TACC(Image)):
return "INVALID, Image \'"+Image+"\' is not provided by TACC"
# Writes the commands to a random file
new_filename = pp.random_file_name()
with open(UPLOAD_FOLDER+new_filename, "w") as comfil:
comfil.write(Image + " /bin/bash -c ")
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
# Creates a new working directory
comfil.write("\"mkdir -p /data; cd /data; ")
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results\"")
comfil.write("\n"+str(TOK))
elif Custom == "No":
comfil.write("\"")
comfil.write(extra_image_commands(Image))
# Adds the files
for FF in Reefil:
comfil.write(get_reef_file(Image, TOK, FF)+" ")
comfil.write(Command+" python /Mov_Res.py\"")
comfil.write("\n"+str(TOK))
# Submits the file for processing
if boapp == "boinc2docker":
toproc(Image, tprov)
shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "adtdp":
shutil.move(UPLOAD_FOLDER+new_filename, ADTDP_FOLDER+new_filename)
return "Commands submitted for processing"
def results_file(FIL, toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
resp = requests.get('http://'+os.environ['Reef_IP']+':2001/reef/results/'+os.environ['Reef_Key']+'/'+toktok+'/'+FIL)
return resp.content
def process_midas_jobs():
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["token"]
tmp_dir = dictdata["folder_name"]
OS = dictdata["operating_system"]
prolangs = dictdata["programming_language"]
libs = dictdata["library_list"]
setfiles = dictdata["setup_filename"]
outfiles = dictdata["output_file"]
coms = dictdata["command_lines"]
except:
return "INVALID, json lacks at least one field"
if pp.token_test(TOK) == False:
shutil.rmtree(TMP)
return 'INVALID, invalid token'
# MIDAS files are stored in a temporary folder
boapp = "boinc2docker"
TMP = "/tmp/" + tmp_dir + '/'
if 'README.txt' in os.listdir(TMP):
return "INVALID, README.txt is a MIDAS reserved file. Please, remove it and try again"
for file in os.listdir(TMP):
if (file.split(".")[-1] == "tgz") or (".".join(
file.split(".")[::-1][:2:][::-1]) == "tar.gz"):
# Takes the contents out and deletes the tar
try:
tar = tarfile.open(TMP + file)
tar.extractall(TMP)
tar.close()
os.remove(TMP + file)
continue
except:
shutil.rmtree(TMP)
return "INVALID, cannot open tar file"
if (file.split(".")[-1] == "zip"):
try:
zip_ref = zipfile.ZipFile(TMP + file, 'r')
zip_ref.extractall(TMP)
zip_ref.close()
os.remove(TMP + file)
except:
shutil.rmtree(TMP)
return "INVALID, cannot open zip file"
# The application is adtdp if using c++ libraries
if libs['cPlusPlus'] != []:
boapp = "adtdp"
# Creates an acceptable README
with open(TMP + "README.txt", 'w') as readme:
try:
readme.write(verne(dictdata, TMP, boapp))
except Exception as e:
shutil.rmtree(TMP)
return "INVALID, " + str(e)
# Validates if the file can be processed
if not mdr.valid_README(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, README in user input'
if not mdr.valid_OS(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, OS is not accepted'
if not mdr.present_input_files(TMP):
shutil.rmtree(TMP)
return 'INVALID, Not all input files for commands are present'
if not mdr.valid_language(TMP + '/README.txt'):
shutil.rmtree(TMP)
return 'INVALID, Language is not accepted'
# Avoids cases where no libraries are needed at all
if (not mdr.install_libraries(TMP + '/README.txt')) and (
mdr.install_libraries(TMP + '/README.txt') != []):
shutil.rmtree(TMP)
return 'INVALID, Language does not support libraries'
# Moves the directory to MIDAS and processes it
ALL_USER_DATA = os.listdir('/root/project/api/sandbox_files/DIR_' +
str(TOK))
while True:
new_MID = 'MID_' + pp.random_dir_name()
if new_MID not in ALL_USER_DATA:
break
MIDAS_PATH = UPLOAD_FOLDER + '/DIR_' + str(TOK) + '/' + new_MID
shutil.copytree(TMP, MIDAS_PATH)
shutil.rmtree(TMP)
# Creates a redis database with syntax {TOKEN}.{MID_DIRECTORY}
r.set(TOK + ';' + new_MID, boapp)
return 'File submitted for processing'
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
# Only for internal use, all other use will return an error
if (request.remote_addr != '0.0.0.0') and (request.remote_addr !=
SERVER_IP):
return "INVALID, API for internal use only"
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
TOK = dictdata["token"][0]
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks if user wants boinc2docker or adtd-p
try:
boapp = dictdata['boapp'][0].lower()
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID application"
except:
return "INVALID, application not provided"
file = request.files['file']
# Files must be JSON
if file.filename == '':
return "INVALID, no file provided"
if file.filename.split('.')[-1] != 'json':
return "File type invalid, only JSON files are acccepted"
# Writes the commands to a random file
new_filename = pp.random_file_name()
file.save(UPLOAD_FOLDER + new_filename)
try:
with open(UPLOAD_FOLDER + new_filename, 'r') as ff:
command_data = json.load(ff)
except:
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, json could not be parsed"
# All JSON files must have the following:
# Image, Image is custom, Command
try:
Image = command_data["Image"]
Custom = command_data["Custom"]
Command = command_data["Command"]
except:
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, json lacks at least one field (keys: Image, Custom, Command)"
with open(UPLOAD_FOLDER + new_filename, "w") as comfil:
comfil.write(Image + "/bin/bash -c ")
Invalid_Custom = False
# Custom images require more work because we must ensure the results will be back
if Custom == "Yes":
comfil.write(
"\"" + Command +
"; mkdir -p /root/shared/results/; mv ./* /root/shared/results\""
)
comfil.write("\n" + str(TOK))
elif Custom == "No":
comfil.write("\"cd /data; " + Command + "; python /Mov_Res.py\"")
comfil.write("\n" + str(TOK))
else:
Invalid_Custom = True
if Invalid_Custom:
# There is no need to keep the file after if has been deleted
os.remove(UPLOAD_FOLDER + new_filename)
return "INVALID, custom value can only be Yes/No"
# Submits the file for processing
if boapp == "boinc2docker":
shutil.move(UPLOAD_FOLDER + new_filename, BOINC_FOLDER + new_filename)
if boapp == "adtdp":
shutil.move(UPLOAD_FOLDER + new_filename, ADTDP_FOLDER + new_filename)
return "Commands submitted for processing"
def upload_file(toktok, Username):
TOK = toktok
if pp.token_test(TOK) == False:
return 'Invalid token'
if request.method != 'POST':
return "Invalid. Submit a text file"
file = request.files['file']
# Gathers all the data
try:
dictdata = ImmutableMultiDict(request.form).to_dict(flat='')
app = str(dictdata["app"][0].lower())
try:
tags_used = [x.strip() for x in dictdata["topics"][0].split(";") if x.strip() != ""]
if tags_used == []:
tags_used = "STEM"
else:
tags_used = ",".join(tags_used)
tags_used = tags_used.lower()
except Exception as e:
print(e)
# Error in processing json
tags_used = "STEM"
except:
return "INVALID, not all data has been provided"
# If no app is provided, it will assume BOINC
try:
if (app != "boinc2docker") and (app != "adtdp"):
return "INVALID app"
except:
app = "boinc2docker"
# Avoids empty files and non-text files
if file.filename == '':
return 'No file submitted'
if file.filename.split('.')[-1] != 'txt':
return "File type invalid, only text files are acccepted"
# Randomizes the file name to avoid matches
new_filename = pp.random_file_name()
file.save(os.path.join(UPLOAD_FOLDER, new_filename))
# Gets the image names
# Although uncommon, users can write multiple commands in a single file
with open(UPLOAD_FOLDER+'/'+new_filename, 'r') as comfil:
for line in comfil:
LL = line.replace('\n', '')
if len(line.split(' ')) == 1:
continue
an_image = line.split(' ')[0]
a_command = " ".join(line.split(' ')[1:])
if image_is_TACC(an_image):
# Gets command
command_itself = a_command.replace('/bin/bash -c "cd /data;', "")
command_itself = command_itself.replace("mv ./* /root/shared/results\"", "")
# Escapes quotes
command_itself = sanitize_str_chars(command_itself)
Complete_command = "/bin/bash -c \""+command_itself+" mv ./* /root/shared/results\""
# Custom image
else:
# Gets command
command_itself = a_command.replace("/bin/bash -c \"", "")
command_itself = a_command.replace("mv ./* /root/shared/results\"", "")
# Escapes quotes
command_itself = sanitize_str_chars(command_itself)
Complete_command = "/bin/bash -c \"mkdir -p /data; cd /data; "+command_itself+" mkdir -p /root/shared/results/; mv ./* /root/shared/results"
# Adds job to database
mints.add_boinc2docker_job(Username, TOK, tags_used, an_image, Complete_command, "boinc2docker", "cli", "Job submitted")
# Removes file
os.remove(UPLOAD_FOLDER+'/'+new_filename)
return "File submitted for processing\n"
def all_user_files(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
resp = requests.get('http://'+os.environ['Reef_IP']+':2000/reef/all_user_files/'+os.environ['Reef_Key']+'/'+toktok)
return resp.text
