def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, FILES_PATH='.'):
researcher_email = pp.obtain_email(UTOK)
try:
user_image(IMTAG)
# Reduces the corresponding user's allocation
# Docker represents image size in GB
imsiz = float(image.get(IMTAG).attrs['Size']) / (10**9)
r.incrbyfloat(UTOK, -imsiz)
# Moves the file
shutil.move(
COMMAND_TXT + ".txt",
"/root/project/html/user/token_data/process_files/" + COMMAND_TXT +
".txt")
# Deletes the key
r.delete(UTOK + '.' + MIDIR)
MESSAGE = Success_Message.replace(
"DATETIME",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE)
except Exception as e:
print(e)
MESSAGE = Failure_Message.replace(
"DATETIME",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, DOCK_DOCK, BOCOM, FILES_PATH='.'):
researcher_email = pp.obtain_email(UTOK)
try:
user_image(IMTAG)
# Reduces the corresponding user's allocation
# Docker represents image size in GB
# Moves the file
boapp = r.get(UTOK+'.'+MIDIR).decode("UTF-8")
if boapp == "boinc2docker":
shutil.move(COMMAND_TXT+".txt", "/root/project/html/user/token_data/process_files/"+COMMAND_TXT+".txt")
if boapp == "adtdp":
shutil.move(COMMAND_TXT+".txt", "/root/project/adtd-protocol/process_files/"+COMMAND_TXT+".txt")
# Deletes the key
r.delete(UTOK+'.'+MIDIR)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image."+hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]+".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to the user's result folders
shutil.move(saved_name, "/root/project/api/sandbox_files/DIR_"+UTOK+"/___RESULTS/"+saved_name)
MESSAGE = Success_Message.replace("DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name
MESSAGE += "\n\nRun the following command on the image: \n"+' '.join(BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n"+DOCK_DOCK
pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE)
except Exception as e:
print(e)
r.delete(UTOK+'.'+MIDIR)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nDockerfile created below: \n\n"+DOCK_DOCK
pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG,
UTOK,
MIDIR,
COMMAND_TXT,
DOCK_DOCK,
BOCOM,
FILES_PATH='.'):
researcher_email = pp.obtain_email(UTOK)
try:
user_image(IMTAG)
# Reduces the corresponding user's allocation
# Docker represents image size in GB
# Moves the file
boapp = r.get(UTOK + ';' + MIDIR).decode("UTF-8")
if boapp == "boinc2docker":
# Reads tag information from json file
# Image is the user ID because:
# | - Traceable
# | - MID is a random token
with open("tag_info.json", 'r') as J:
TAGS = json.load(J)
cus.complete_tag_work(UTOK, TAGS)
shutil.move(
COMMAND_TXT + ".txt",
"/home/boincadm/project/html/user/token_data/process_files/" +
COMMAND_TXT + ".txt")
if boapp == "adtdp":
shutil.move(
COMMAND_TXT + ".txt",
"/home/boincadm/project/adtd-protocol/process_files/" +
COMMAND_TXT + ".txt")
# Deletes the key
r.delete(UTOK + ';' + MIDIR)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image." + hashlib.sha256(
str(datetime.datetime.now()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' + os.environ['Reef_Key'] +
'/' + UTOK,
files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace(
"DATETIME",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE)
except Exception as e:
print(e)
r.delete(UTOK + ';' + MIDIR)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace(
"DATETIME",
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK
pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG, UTOK, MIDIR, DOCK_DOCK, BOCOM, FILES_PATH, boapp,
job_id):
researcher_email = pp.obtain_email(UTOK)
# Updates job status
mints.update_job_status(job_id, boapp, "Building image")
try:
user_image(IMTAG)
# VolCon instructions
# Deletes the image, submits the saved version to a mirror
if boapp == "volcon":
# Saves the image into a file
img = image.get(IMTAG)
resp = img.save()
random_generated_dir = hashlib.sha256(
str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]
image_dir = os.getcwd() + "/" + random_generated_dir
os.mkdir(image_dir)
full_image_path = image_dir + "/image.tar.gz"
with open(full_image_path, "wb") as ff:
for salmon in resp:
ff.write(salmon)
VolCon_ID = uuid.uuid4().hex
mirror_IP = mirror.get_random_mirror()
# Move image to the mirror
mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID)
# Moves the file to where it belongs
saved_name = "image_" + hashlib.sha256(
str(datetime.datetime.utcnow()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
shutil.move(full_image_path, saved_name)
# Move commands to mirror
Commands = BOCOM
# Add job to VolCon
# Set as medium priority
GPU_needed = 0
mints.make_MIDAS_job_available(job_id,
"CUSTOM",
Commands,
GPU_needed,
VolCon_ID,
"Middle",
public=0)
mints.update_mirror_ip(VolCon_ID, mirror_IP)
# MIDAS cannot accept GPU jobs
job_info = {
"Image": "Custom",
"Command": Commands,
"TACC": 0,
"GPU": 0,
"VolCon_ID": VolCon_ID,
"public": 0,
"key": mirror.mirror_key(mirror_IP)
}
requests.post(
'http://' + mirror_IP +
":7000/volcon/mirror/v2/api/public/receive_job_files",
json=job_info)
# Moves data to Reef
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' +
os.environ['Reef_Key'] + '/' + UTOK,
files={"file": open(saved_name, "rb")})
# Deletes local copy
os.remove(saved_name)
# Removes the image
container.prune()
image.remove(IMTAG, force=True)
# Email user with dockerfile
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
return None
# Updates the database so that the MIDAS job can be processed
mints.make_boinc2docker_MIDAS_job_available(job_id, IMTAG.lower(),
BOCOM)
# Saves the docker image and sends the user the dockerfile and a link to the tar ball
# docker-py documentation was erronous
img = image.get(IMTAG)
resp = img.save()
# Creates a file, recycled everytime the program runs
saved_name = "image." + hashlib.sha256(
str(datetime.datetime.now()).encode(
'UTF-8')).hexdigest()[:4:] + ".tar.gz"
ff = open(saved_name, 'wb')
for salmon in resp:
ff.write(salmon)
ff.close()
# Moves the file to reef and deletes the local copy
requests.post('http://' + os.environ['Reef_IP'] +
':2001/reef/result_upload/' + os.environ['Reef_Key'] +
'/' + UTOK,
files={"file": open(saved_name, "rb")})
os.remove(saved_name)
MESSAGE = Success_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n"
MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n"
MESSAGE += os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name
MESSAGE += "\n\nDownload the image using the following command:\n"
MESSAGE += "curl -O " + os.environ[
"SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace(
"../", "")
MESSAGE += "\nThen load the image (sudo permission may be required):"
MESSAGE += "\ndocker load < " + saved_name.replace("../", "")
MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):"
MESSAGE += "\ndocker run -it IMAGE_ID bash"
MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join(
BOCOM.split(' ')[1::])
MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Succesful MIDAS build",
MESSAGE, [])
except Exception as e:
print(e)
# Updates status and notified time
mints.update_job_status_notified(job_id,
boapp,
"Error creating MIDAS Dockerfile",
notified_date_provided=False,
processing_error=None)
# Deletes the unused container
client.containers.prune()
MESSAGE = Failure_Message.replace("DATETIME", mints.timnow())
MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK
ec.send_mail_complete(researcher_email, "Failed MIDAS build", MESSAGE,
[])