def upload_job_to_mirror(JOB_INFO):
mirror_ip = get_random_mirror()
JOB_INFO["key"] = mirror_key(get_random_mirror())
mints.update_mirror_ip(JOB_INFO["VolCon_ID"], mirror_ip)
# Updates result to mirror
requests.post('http://'+mirror_ip+":7000/volcon/mirror/v2/api/public/receive_job_files",
json=JOB_INFO)
def upload_job_to_mirror(JOB_INFO):
mirror_ip = get_random_mirror()
JOB_INFO["key"] = mirror_key(get_random_mirror())
mints.update_mirror_ip(JOB_INFO["VolCon_ID"], mirror_ip)
# Updates result to mirror
requests.post('http://' + mirror_ip +
":7000/volcon/mirror/v2/api/public/receive_job_files",
json=JOB_INFO)
# Calls own server to update the new status
requests.post('http://' + os.environ["SERVER_IP"] +
":5089/volcon/v2/api/mirrors/status/update",
json=JOB_INFO)
def complete_build(IMTAG, UTOK, MIDIR, DOCK_DOCK, BOCOM, FILES_PATH, boapp,
job_id):
researcher_email = pp.obtain_email(UTOK)
# Updates job status
mints.update_job_status(job_id, boapp, "Building image")
try:
user_image(IMTAG)
# VolCon instructions
# Deletes the image, submits the saved version to a mirror
if boapp == "volcon":
# Saves the image into a file
img = image.get(IMTAG)
resp = img.save()
random_generated_dir = hashlib.sha256(
str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]
image_dir = os.getcwd() + "/" + random_generated_dir
os.mkdir(image_dir)
full_image_path = image_dir + "/image.tar.gz"
with open(full_image_path, "wb") as ff:
for salmon in resp:
ff.write(salmon)
VolCon_ID = uuid.uuid4().hex
mirror_IP = mirror.get_random_mirror()
# Move image to the mirror
mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID)
# Moves the file to where it belongs
saved_name = "image_" + hashlib.sha256(
str(datetime.datetime.utcnow()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
shutil.move(full_image_path, saved_name)
# Move commands to mirror
Commands = BOCOM
# Add job to VolCon
# Set as medium priority
GPU_needed = 0
mints.make_MIDAS_job_available(job_id,
"CUSTOM",
Commands,
GPU_needed,
VolCon_ID,
"Middle",
public=0)
mints.update_mirror_ip(VolCon_ID, mirror_IP)
# MIDAS cannot accept GPU jobs
job_info = {
"Image": "Custom",
"Command": Commands,
"TACC": 0,
"GPU": 0,
"VolCon_ID": VolCon_ID,
"public": 0,
"key": mirror.mirror_key(mirror_IP)
}
requests.post(
'http://' + mirror_IP +
":7000/volcon/mirror/v2/api/public/receive_job_files",
json=job_info)
# Moves data to Reef
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' +
os.environ['Reef_Key'] + '/' + UTOK,
files={"file": open(saved_name, "rb")})
# Deletes local copy
os.remove(saved_name)
# Removes the image
container.prune()
image.remove(IMTAG, force=True)
# Email user with dockerfile
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
return None
# Updates the database so that the MIDAS job can be processed
mints.make_boinc2docker_MIDAS_job_available(job_id, IMTAG.lower(),
BOCOM)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image." + hashlib.sha256(
str(datetime.datetime.now()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' + os.environ['Reef_Key'] +
'/' + UTOK,
files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
except Exception as e:
print(e)
# Updates status and notified time
mints.update_job_status_notified(job_id,
boapp,
"Error creating MIDAS Dockerfile",
notified_date_provided=False,
processing_error=None)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Failed MIDAS build", MESSAGE,
[])