def failed_report():
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
req_fields = ["date (Run)", "VolCon-ID", "download time", "Error"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in req_check])
VolCon_ID = proposal["VolCon-ID"]
if not mints.VolCon_ID_exists(VolCon_ID):
return "INVALID: VolCon-ID does not exist"
date_run = proposal["date (Run)"]
download_time = proposal["download time"]
received_time = mints.timnow()
Error = proposal["Error"]
outcome = Error
user_token = mints.token_from_VolCon_ID(VolCon_ID)
researcher_email = email_from_token(user_token)
# Sends email to user, no attachments since the job did not run
email_notification = ec.send_mail_complete(researcher_email, "BOINC job failed", ec.automatic_text(received_time, outcome, user_token, []),
[])
client_IP = request.environ['REMOTE_ADDR']
# Updates the database
mints.failed_execution_report(VolCon_ID, date_run, download_time, Error, email_notification, client_IP)
return "Server has processed the results"
def results_upload(VID):
prestime = mints.timnow()
# Checks out if the given VolCon-ID exists in database
if not mints.VolCon_ID_exists(VID):
return "INVALID: VolCon-ID does not exist"
try:
file = request.files["file"]
except:
return "INVALID, file not provided"
# Always in the same location
location = "/results/volcon/"+present_day()
# Creates directory if needed
if present_day() not in os.listdir("/results/volcon"):
# Creates directory, avoids race conditions
try:
os.mkdir(location)
except:
pass
new_name = secure_filename(file.filename)
# Saves the file
file.save(location+"/"+new_name)
# Saves the location of the file
mints.update_results_path_apache(VID, location+"/"+new_name)
# Updates the status in the database
mints.update_job_status(VID, "Results received", False)
return "Results uploaded"
def upload_report():
if not request.is_json:
return "INVALID: Request is not json"
proposal = request.get_json()
# Checks the required fields
req_fields = ["date (Run)", "VolCon-ID", "download time", "Commands", "Result Error", "computation time"]
req_check = l2_contains_l1(req_fields, proposal.keys())
if req_check != []:
return "INVALID: Lacking the following json fields to be read: "+",".join([str(a) for a in req_check])
VolCon_ID = proposal["VolCon-ID"]
if not mints.VolCon_ID_exists(VolCon_ID):
return "INVALID: VolCon-ID does not exist"
command_errors = proposal["Commands"][1]
computation_time = proposal["computation time"]
date_run = proposal["date (Run)"]
download_time = proposal["download time"]
received_time = mints.timnow()
result_error = proposal["Result Error"]
user_token = mints.token_from_VolCon_ID(VolCon_ID)
researcher_email = email_from_token(user_token)
Error = "" # By default
if set(command_errors) != {"Success"}:
Error = ",".join([str(x) for x, y in zip(range(0, len(command_errors)), command_errors) if y != "Success" ])
if result_error != "0":
Error += ";"+result_error
# No attachments can be added since none where updated
attachments = []
else:
# Finds the attachments, in chronological order starting at a date
attachments = mints.read_results_path_apache(VolCon_ID)
result_error = "No errors retrieving data"
if Error == "":
Error = None
outcome = "Success"
specific_command_errors = None
# Uploads the data to Reef
requests.post('http://'+os.environ['Reef_IP']+':2001/reef/result_upload/'+os.environ['Reef_Key']+'/'+user_token,
files={"file": open(attachments[0], "rb")})
mints.update_results_path_reef(VolCon_ID, attachments[0])
else:
# Types of error
outcome = "Computational error"
attachments = []
specific_command_errors = ",".join(command_errors)+";"+result_error
# Sends email to user
email_notification = ec.send_mail_complete(researcher_email, "BOINC job complete", ec.automatic_text(received_time, outcome, user_token, attachments),
attachments)
client_IP = request.environ['REMOTE_ADDR']
# Updates the database
mints.update_execution_report(VolCon_ID, specific_command_errors, computation_time, date_run, download_time,
Error, email_notification, client_IP)
return "Server has processed the results"
def complete_build(IMTAG, UTOK, MIDIR, DOCK_DOCK, BOCOM, FILES_PATH, boapp,
job_id):
researcher_email = pp.obtain_email(UTOK)
# Updates job status
mints.update_job_status(job_id, boapp, "Building image")
try:
user_image(IMTAG)
# VolCon instructions
# Deletes the image, submits the saved version to a mirror
if boapp == "volcon":
# Saves the image into a file
img = image.get(IMTAG)
resp = img.save()
random_generated_dir = hashlib.sha256(
str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]
image_dir = os.getcwd() + "/" + random_generated_dir
os.mkdir(image_dir)
full_image_path = image_dir + "/image.tar.gz"
with open(full_image_path, "wb") as ff:
for salmon in resp:
ff.write(salmon)
VolCon_ID = uuid.uuid4().hex
mirror_IP = mirror.get_random_mirror()
# Move image to the mirror
mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID)
# Moves the file to where it belongs
saved_name = "image_" + hashlib.sha256(
str(datetime.datetime.utcnow()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
shutil.move(full_image_path, saved_name)
# Move commands to mirror
Commands = BOCOM
# Add job to VolCon
# Set as medium priority
GPU_needed = 0
mints.make_MIDAS_job_available(job_id,
"CUSTOM",
Commands,
GPU_needed,
VolCon_ID,
"Middle",
public=0)
mints.update_mirror_ip(VolCon_ID, mirror_IP)
# MIDAS cannot accept GPU jobs
job_info = {
"Image": "Custom",
"Command": Commands,
"TACC": 0,
"GPU": 0,
"VolCon_ID": VolCon_ID,
"public": 0,
"key": mirror.mirror_key(mirror_IP)
}
requests.post(
'http://' + mirror_IP +
":7000/volcon/mirror/v2/api/public/receive_job_files",
json=job_info)
# Moves data to Reef
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' +
os.environ['Reef_Key'] + '/' + UTOK,
files={"file": open(saved_name, "rb")})
# Deletes local copy
os.remove(saved_name)
# Removes the image
container.prune()
image.remove(IMTAG, force=True)
# Email user with dockerfile
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
return None
# Updates the database so that the MIDAS job can be processed
mints.make_boinc2docker_MIDAS_job_available(job_id, IMTAG.lower(),
BOCOM)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image." + hashlib.sha256(
str(datetime.datetime.now()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' + os.environ['Reef_Key'] +
'/' + UTOK,
files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
except Exception as e:
print(e)
# Updates status and notified time
mints.update_job_status_notified(job_id,
boapp,
"Error creating MIDAS Dockerfile",
notified_date_provided=False,
processing_error=None)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Failed MIDAS build", MESSAGE,
[])