def tacc_jobs():
# Ensures that there is an appropriate json request
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
if pp.token_test(proposal["token"]) == False:
return 'Invalid token'
# Checks the required fields
req_fields = ["token", "image", "commands", "priority", "username"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: " + ",".join(
[str(a) for a in req_check])
[TOKEN, IMAGE, COMMANDS, PRIORITY, USERNAME] = [
proposal["token"], proposal["image"], proposal["commands"],
proposal["priority"], proposal["username"]
]
VolCon_ID = uuid.uuid4().hex
if "gpu" in IMAGE:
GPU = 1
else:
GPU = 0
try:
mints.add_job(TOKEN,
IMAGE,
COMMANDS,
GPU,
VolCon_ID,
PRIORITY,
username=USERNAME)
except:
return "INVALID: Could not connect to MySQL database"
# TACC: Image is a TACC image
job_info = {
"Image": IMAGE,
"Command": COMMANDS,
"TACC": 1,
"GPU": GPU,
"VolCon_ID": VolCon_ID,
"public": 1
}
try:
mirror.upload_job_to_mirror(job_info)
except:
return "No VolCon mirrors are connected to this project"
return "Successfully submitted job"
def new_job():
if not request.is_json:
return "POST parameters could not be parsed"
ppr = request.get_json()
ppr_keys = ppr.keys()
check = l2_contains_l1(["key", "ID", "User", "origin", "Job", "modules", "output_files", "dirname", "sc_system",
"sc_queue", "n_cores", "n_nodes", "runtime"], ppr_keys)
if check:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check])
key = ppr["key"]
username = ppr["User"]
# Allowed access for:
# - Systems with orchestra key, springIPT mainly
# - Calls from wetty where the user is a TACC user
invalid_access = True
if valid_adm_passwd(key):
invalid_access = False
else:
reqip = IP_to_hostname(request.environ['REMOTE_ADDR'])
r_occupied = redis.Redis(host=URL_BASE, port=6379, password=REDIS_AUTH, db=0)
# Valid IP and user is in the IP
if (reqip in redkeys(r_occupied)) and (not (mints.get_ip_port(username, reqip)[1])):
invalid_access = False
if invalid_access:
return "INVALID: Access not allowed"
job_ID = ppr["ID"]
origin = ppr["origin"]
job_type = ppr["Job"]
modules = ppr["modules"]
output_files = ppr["output_files"]
dirname = ppr["dirname"]
sc_system = ppr["sc_system"]
sc_queue = ppr["sc_queue"]
n_cores = ppr["n_cores"]
n_nodes = ppr["n_nodes"]
runtime = ppr["runtime"]
# If the user is not TACC, job cannot be submitted
[type_of_user, error_in_db] = mints.current_user_status(username)
greyfish_commonuser_job_loc = "/greyfish/sandbox/DIR_commonuser/jobs_left/"+dirname+".zip"
if error_in_db:
# Deletes the job data if it exists
if os.path.exists(greyfish_commonuser_job_loc):
os.remove(greyfish_commonuser_job_loc)
return "User is not logged in correctly"
if type_of_user[0] == "false":
# Deletes the job data if it exists
if os.path.exists(greyfish_commonuser_job_loc):
os.remove(greyfish_commonuser_job_loc)
return "User is not authorized to submit jobs"
if hhmmss_pattern.match(runtime) == None:
return "INVALID: time mut be specified as HH:MM:SS"
if job_type not in ["Compile", "Run", "Both"]:
return "INVALID: Job type not accepted, must be 'Compile', 'Run', or 'Both'"
# Processes the commands
if job_type == "Compile":
check_compile = l2_contains_l1(["CC"], ppr_keys)
if check_compile:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_compile])
number_compile_instructions = int(ppr["CC"])
compile_instruction_tags = ["C"+str(c) for c in range(0, number_compile_instructions)]
check_compile2 = l2_contains_l1(compile_instruction_tags, ppr_keys)
if check_compile2:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_compile2])
compile_instructions = [ppr[c_tag] for c_tag in compile_instruction_tags]
run_instructions = None
elif job_type == "Run":
check_run = l2_contains_l1(["RC"], ppr_keys)
if check_run:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_run])
number_run_instructions = int(ppr["RC"])
run_instruction_tags = ["R"+str(c) for c in range(0, number_run_instructions)]
check_run2 = l2_contains_l1(run_instruction_tags, ppr_keys)
if check_run2:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_run2])
compile_instructions = None
run_instructions = [ppr[r_tag] for r_tag in run_instruction_tags]
elif job_type == "Both":
check_both = l2_contains_l1(["CC", "RC"], ppr_keys)
if check_both:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_compile])
number_compile_instructions = int(ppr["CC"])
compile_instruction_tags = ["C"+str(c) for c in range(0, number_compile_instructions)]
check_compile2 = l2_contains_l1(compile_instruction_tags, ppr_keys)
if check_compile2:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_compile2])
compile_instructions = [ppr[c_tag] for c_tag in compile_instruction_tags]
number_run_instructions = int(ppr["RC"])
run_instruction_tags = ["R"+str(c) for c in range(0, number_run_instructions)]
check_run2 = l2_contains_l1(run_instruction_tags, ppr_keys)
if check_run2:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check_run2])
run_instructions = [ppr[r_tag] for r_tag in run_instruction_tags]
# Adds the data to a json file if using the web interface
# Warning, only the directory with same name will be added
if origin == "web":
GREYFISH_DIR = "/greyfish/sandbox/DIR_commonuser/jobs_left/"
zip_location = GREYFISH_DIR+dirname+".zip"
os.mkdir(GREYFISH_DIR+dirname)
os.chdir(GREYFISH_DIR+dirname)
with zipfile.ZipFile(zip_location, 'r') as zf:
zf.extractall(".")
# Must have a directory with the same name as the zipfile
dirs_inside = [adir for adir in os.listdir('.') if os.path.isdir(GREYFISH_DIR+dirname+"/"+adir)]
if len(dirs_inside) != 1:
return "INVALID: "+str(len(dirs_inside)+" were provided, only one directory containing all the necessary data can be inside the zip file")
only_dir_with_info = dirs_inside[0]
os.rename(only_dir_with_info, dirname)
for item in os.listdir('.'):
if item == dirname:
continue
if os.path.isdir(item):
shutil.rmtree(item)
else:
# for files
os.remove(item)
web_data_to_json_file.json_to_file(dirname, ppr)
os.remove(zip_location)
shutil.make_archive(GREYFISH_DIR+dirname, "zip", ".")
os.chdir(CURDIR)
shutil.rmtree(GREYFISH_DIR+dirname)
mints.add_job(job_ID, username, compile_instructions, run_instructions, job_type, origin, modules, output_files, dirname,
sc_system, sc_queue, n_cores, n_nodes)
return "New job added to database"
def process_web_jobs():
if request.method != 'POST':
return "INVALID, no data provided"
try:
dictdata = request.get_json()
except:
return "INVALID, JSON could not be parsed"
try:
TOK = dictdata["Token"]
Reefil = dictdata["Files"]
Image = dictdata["Image"]
Custom = dictdata["Custom"]
Command = dictdata["Command"]
Username = dictdata["Username"]
if "priority" not in dictdata.keys():
PRIORITY = "Middle"
else:
PRIORITY = dictdata["priority"]
except:
return "INVALID, json lacks at least one field (keys: Token, Boapp, Files, Image, Custom, Command, Username)"
if pp.token_test(TOK) == False:
return "INVALID token"
# Checks the list of Commands to ensure their correct execution
Command = ';'.join([C for C in Command.split(';') if (C != '') and (C.count(C[0]) != len(C)) ]) +';'
boapp = "boinc2docker"
if Image == "carlosred/gpu:cuda":
boapp = "volcon"
if (Custom != "Yes" ) and (Custom != "No"):
return abort(422) # Incorrect API
if not image_is_TACC(Image):
Custom = "Yes"
# Gets the tags
try:
tags_used = [x.strip() for x in dictdata["topics"].split(";") if x.strip() != ""]
if tags_used == []:
tags_used = "STEM"
else:
tags_used = ",".join(tags_used)
tags_used = tags_used.lower()
except Exception as e:
print(e)
# Error in processing json
tags_used = "STEM"
else:
# Default tag: STEM
tags_used = "STEM"
if boapp == "boinc2docker":
# Writes the commands to a random file
new_filename = pp.random_file_name()
Complete_command = ""
# Custom images require more work because it must be ensured the results will be back
if Custom == "Yes":
# Creates a new working directory
Complete_command += "mkdir -p /data; cd /data; "
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command+" mkdir -p /root/shared/results/; mv ./* /root/shared/results"
elif Custom == "No":
# Adds the files
for FF in Reefil:
# Skips unnecessary files
if FF == "":
break
Complete_command += get_reef_file(Image, TOK, FF)+" "
Complete_command += Command +" mv ./* /root/shared/results"
# Replaces certain characters
Complete_command = " /bin/bash -c \""+sanitize_str_chars(Complete_command)+"\""
mints.add_boinc2docker_job(Username, TOK, tags_used, Image, Complete_command, boapp, "web", "Job submitted")
# Old way
# shutil.move(UPLOAD_FOLDER+new_filename, BOINC_FOLDER+new_filename)
if boapp == "volcon":
# Only CUDA requires a GPU
# Custom images are also assumed to not require a GPU TODO TODO TODO TODO
if Image == "carlosred/gpu:cuda":
GPU = 1
else:
GPU = 0
VolCon_ID = uuid.uuid4().hex
COMMANDS = ""
if Custom == "Yes":
TACC = 0
COMMANDS += "mkdir -p /data; cd /data; "
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
# Splits the commands and ensures that they are run in /data
newcoms = ";".join(["cd /data && "+x for x in Command.split(";")])
COMMANDS += newcoms+" mkdir -p /root/shared/results/; mv /data/* /root/shared/results"
else:
TACC = 1
for FF in Reefil:
if FF == '':
break
COMMANDS += get_reef_file(Image, TOK, FF)+" "
COMMANDS += ";".join([extra_image_commands(Image) +z for z in Command.split(";")])+" mv ./* /root/shared/results"
COMMANDS = " /bin/bash -c \""+COMMANDS+"\""
job_info = {"Image":Image, "Command":COMMANDS, "TACC":TACC, "GPU":GPU, "VolCon_ID":VolCon_ID, "public":1}
# Updates the job in the database
mints.add_job(TOK, Image, COMMANDS, GPU, VolCon_ID, PRIORITY, 1, tags_used, Username, "web")
# Pushes job information to mirrors
mirror.upload_job_to_mirror(job_info)
return "Commands submitted for processing"
def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, DOCK_DOCK, BOCOM, FILES_PATH='.'):
researcher_email = pp.obtain_email(UTOK)
try:
user_image(IMTAG)
# Reduces the corresponding user's allocation
# Docker represents image size in GB
# Moves the file
boapp = r.get(UTOK+';'+MIDIR).decode("UTF-8")
if boapp == "boinc2docker":
shutil.move(COMMAND_TXT+".txt", "/home/boincadm/project/html/user/token_data/process_files/"+COMMAND_TXT+".txt")
# VolCon instructions
# Deletes the image, submits the saved version to a mirror
if boapp == "volcon":
# Deletes the key
r.delete(UTOK+';'+MIDIR)
# Saves the image into a file
img = image.get(IMTAG)
resp = img.save()
random_generated_dir = hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]
image_dir = os.getcwd()+"/"+random_generated_dir
os.mkdir(image_dir)
full_image_path = image_dir+"/image.tar.gz"
with open(full_image_path, "wb") as ff:
for salmon in resp:
ff.write(salmon)
VolCon_ID = uuid.uuid4().hex
mirror_IP = mirror.get_random_mirror()
# Move image to the mirror
mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID)
# Moves the file to where it belongs
saved_name = "image_"+hashlib.sha256(str(datetime.datetime.utcnow()).encode('UTF-8')).hexdigest()[:4:]+".tar.gz"
shutil.move(full_image_path, saved_name)
# Move commands to mirror
Commands = " ".join(BOCOM.split(" ")[1::])
# Add job to VolCon
# Set as medium priority
mints.add_job(UTOK, "Custom", Commands, 0, VolCon_ID, "Middle", public=0)
mints.update_mirror_ip(VolCon_ID, mirror_IP)
# MIDAS cannot accept GPU jobs
job_info = {"Image":"Custom", "Command":Commands, "TACC":0, "GPU":0,
"VolCon_ID":VolCon_ID, "public":0, "key":mirror.mirror_key(mirror_IP)}
requests.post('http://'+mirror_IP+":7000/volcon/mirror/v2/api/public/receive_job_files",
json=job_info)
# Moves data to Reef
requests.post('http://'+os.environ['Reef_IP']+':2001/reef/result_upload/'+os.environ['Reef_Key']+'/'+UTOK,
files={"file": open(saved_name, "rb")})
# Deletes local copy
os.remove(saved_name)
# Removes the image
container.prune()
image.remove(IMTAG, force=True)
# Email user with dockerfile
MESSAGE = Success_Message.replace("DATETIME", datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name.replace("../", "")
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O "+os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name.replace("../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < "+saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n"+' '.join(BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n"+DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build", MESSAGE, [])
return None
# Deletes the key
r.delete(UTOK+';'+MIDIR)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image."+hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]+".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://'+os.environ['Reef_IP']+':2001/reef/result_upload/'+os.environ['Reef_Key']+'/'+UTOK, files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace("DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O "+os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name.replace("../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < "+saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n"+' '.join(BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n"+DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build", MESSAGE, [])
except Exception as e:
print(e)
r.delete(UTOK+';'+MIDIR)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nDockerfile created below: \n\n"+DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build", MESSAGE, [])