def results_upload(VID):
prestime = mints.timnow()
# Checks out if the given VolCon-ID exists in database
if not mints.VolCon_ID_exists(VID):
return "INVALID: VolCon-ID does not exist"
try:
file = request.files["file"]
except:
return "INVALID, file not provided"
# Always in the same location
location = "/results/volcon/"+present_day()
# Creates directory if needed
if present_day() not in os.listdir("/results/volcon"):
# Creates directory, avoids race conditions
try:
os.mkdir(location)
except:
pass
new_name = secure_filename(file.filename)
# Saves the file
file.save(location+"/"+new_name)
# Saves the location of the file
mints.update_results_path_apache(VID, location+"/"+new_name)
# Updates the status in the database
mints.update_job_status(VID, "Results received", False)
return "Results uploaded"
def status_update():
# Ensures that there is an appropriate json request
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
req_fields = ["key", "VolCon-ID", "status"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: " + ",".join(
[str(a) for a in req_check])
if bad_password("VolCon", proposal["key"]):
return "INVALID: incorrect password"
try:
mints.update_job_status(proposal["VolCon-ID"], proposal["status"],
True)
return "Successfully updated job " + proposal["VolCon-ID"]
except:
return "Failed to update job"
def status_update():
if not request.is_json:
return "POST parameters could not be parsed"
ppr = request.get_json()
ppr_keys = ppr.keys()
check = l2_contains_l1(["key", "job_ID", "status", "error"], ppr_keys)
if check:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in check])
key = ppr["key"]
job_ID = ppr["job_ID"]
status = ppr["status"]
error = ppr["error"]
if not valid_adm_passwd(key):
return "INVALID: Access not allowed"
if error == "":
error = None
mints.update_job_status(job_ID, status, error)
return "Updated job status"
def request_job():
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
req_fields = ["cluster", "disconnect-key", "GPU", "priority-level"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in req_check])
# By default, it assumes public jobs, change it to 0 to account for MIDAS
wants_public_jobs = 1
if "public" in proposal.keys():
wants_public_jobs = proposal["public"]
# Ensures the VolCon client is associated with a valid cluster
IP = request.environ['REMOTE_ADDR']
cluster = proposal["cluster"]
if not r.hexists(cluster, cluster+"-"+IP):
return "INVALID: Server IP is not associated with cluster ' "+cluster+ "'"
# Ensures that the key provided for this particular server is correct
if r.hget(cluster, cluster+"-"+IP).decode("UTF-8") != proposal["disconnect-key"]:
return "INVALID key"
# Obtains a list of valid volcon_IDs and their respective mirror IPs
volmir = mints.available_jobs(proposal["GPU"], proposal["priority-level"], public=wants_public_jobs)
random.shuffle(volmir)
if volmir == []:
return jsonify({"jobs-available":"0"})
# Locks and selects the first one for execution to avoid race conditions
unavailable = True
for item in volmir:
VID = item[0]
new_status = "Job has been requested by client"
if not mints.race_condition_occurred(VID):
try:
mints.update_job_status(VID, new_status, True)
except:
continue
VolCon_ID = VID
mirror_IP = item[1]
unavailable = False
break
if unavailable:
return jsonify({"jobs-available":"0"})
# Finds the mirror location of the files
return jsonify({"VolCon-ID":VolCon_ID, "mirror-IP":mirror_IP})
def complete_build(IMTAG, UTOK, MIDIR, DOCK_DOCK, BOCOM, FILES_PATH, boapp,
job_id):
researcher_email = pp.obtain_email(UTOK)
# Updates job status
mints.update_job_status(job_id, boapp, "Building image")
try:
user_image(IMTAG)
# VolCon instructions
# Deletes the image, submits the saved version to a mirror
if boapp == "volcon":
# Saves the image into a file
img = image.get(IMTAG)
resp = img.save()
random_generated_dir = hashlib.sha256(
str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]
image_dir = os.getcwd() + "/" + random_generated_dir
os.mkdir(image_dir)
full_image_path = image_dir + "/image.tar.gz"
with open(full_image_path, "wb") as ff:
for salmon in resp:
ff.write(salmon)
VolCon_ID = uuid.uuid4().hex
mirror_IP = mirror.get_random_mirror()
# Move image to the mirror
mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID)
# Moves the file to where it belongs
saved_name = "image_" + hashlib.sha256(
str(datetime.datetime.utcnow()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
shutil.move(full_image_path, saved_name)
# Move commands to mirror
Commands = BOCOM
# Add job to VolCon
# Set as medium priority
GPU_needed = 0
mints.make_MIDAS_job_available(job_id,
"CUSTOM",
Commands,
GPU_needed,
VolCon_ID,
"Middle",
public=0)
mints.update_mirror_ip(VolCon_ID, mirror_IP)
# MIDAS cannot accept GPU jobs
job_info = {
"Image": "Custom",
"Command": Commands,
"TACC": 0,
"GPU": 0,
"VolCon_ID": VolCon_ID,
"public": 0,
"key": mirror.mirror_key(mirror_IP)
}
requests.post(
'http://' + mirror_IP +
":7000/volcon/mirror/v2/api/public/receive_job_files",
json=job_info)
# Moves data to Reef
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' +
os.environ['Reef_Key'] + '/' + UTOK,
files={"file": open(saved_name, "rb")})
# Deletes local copy
os.remove(saved_name)
# Removes the image
container.prune()
image.remove(IMTAG, force=True)
# Email user with dockerfile
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
return None
# Updates the database so that the MIDAS job can be processed
mints.make_boinc2docker_MIDAS_job_available(job_id, IMTAG.lower(),
BOCOM)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image." + hashlib.sha256(
str(datetime.datetime.now()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' + os.environ['Reef_Key'] +
'/' + UTOK,
files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
except Exception as e:
print(e)
# Updates status and notified time
mints.update_job_status_notified(job_id,
boapp,
"Error creating MIDAS Dockerfile",
notified_date_provided=False,
processing_error=None)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Failed MIDAS build", MESSAGE,
[])