def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, FILES_PATH='.'): researcher_email = pp.obtain_email(UTOK) try: user_image(IMTAG) # Reduces the corresponding user's allocation # Docker represents image size in GB imsiz = float(image.get(IMTAG).attrs['Size']) / (10**9) r.incrbyfloat(UTOK, -imsiz) # Moves the file shutil.move( COMMAND_TXT + ".txt", "/root/project/html/user/token_data/process_files/" + COMMAND_TXT + ".txt") # Deletes the key r.delete(UTOK + '.' + MIDIR) MESSAGE = Success_Message.replace( "DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE) except Exception as e: print(e) MESSAGE = Failure_Message.replace( "DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, DOCK_DOCK, BOCOM, FILES_PATH='.'): researcher_email = pp.obtain_email(UTOK) try: user_image(IMTAG) # Reduces the corresponding user's allocation # Docker represents image size in GB # Moves the file boapp = r.get(UTOK+'.'+MIDIR).decode("UTF-8") if boapp == "boinc2docker": shutil.move(COMMAND_TXT+".txt", "/root/project/html/user/token_data/process_files/"+COMMAND_TXT+".txt") if boapp == "adtdp": shutil.move(COMMAND_TXT+".txt", "/root/project/adtd-protocol/process_files/"+COMMAND_TXT+".txt") # Deletes the key r.delete(UTOK+'.'+MIDIR) # Saves the docker image and sends the user the dockerfile and a link to the tar ball # docker-py documentation was erronous img = image.get(IMTAG) resp = img.save() # Creates a file, recycled everytime the program runs saved_name = "image."+hashlib.sha256(str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:]+".tar.gz" ff = open(saved_name, 'wb') for salmon in resp: ff.write(salmon) ff.close() # Moves the file to the user's result folders shutil.move(saved_name, "/root/project/api/sandbox_files/DIR_"+UTOK+"/___RESULTS/"+saved_name) MESSAGE = Success_Message.replace("DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n" MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n" MESSAGE += os.environ["SERVER_IP"]+":5060/boincserver/v2/reef/results/"+UTOK+"/"+saved_name MESSAGE += "\n\nRun the following command on the image: \n"+' '.join(BOCOM.split(' ')[1::]) MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n"+DOCK_DOCK pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE) except Exception as e: print(e) r.delete(UTOK+'.'+MIDIR) # Deletes the unused container client.containers.prune() MESSAGE = Failure_Message.replace("DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) MESSAGE += "\n\nDockerfile created below: \n\n"+DOCK_DOCK pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG, UTOK, MIDIR, COMMAND_TXT, DOCK_DOCK, BOCOM, FILES_PATH='.'): researcher_email = pp.obtain_email(UTOK) try: user_image(IMTAG) # Reduces the corresponding user's allocation # Docker represents image size in GB # Moves the file boapp = r.get(UTOK + ';' + MIDIR).decode("UTF-8") if boapp == "boinc2docker": # Reads tag information from json file # Image is the user ID because: # | - Traceable # | - MID is a random token with open("tag_info.json", 'r') as J: TAGS = json.load(J) cus.complete_tag_work(UTOK, TAGS) shutil.move( COMMAND_TXT + ".txt", "/home/boincadm/project/html/user/token_data/process_files/" + COMMAND_TXT + ".txt") if boapp == "adtdp": shutil.move( COMMAND_TXT + ".txt", "/home/boincadm/project/adtd-protocol/process_files/" + COMMAND_TXT + ".txt") # Deletes the key r.delete(UTOK + ';' + MIDIR) # Saves the docker image and sends the user the dockerfile and a link to the tar ball # docker-py documentation was erronous img = image.get(IMTAG) resp = img.save() # Creates a file, recycled everytime the program runs saved_name = "image." + hashlib.sha256( str(datetime.datetime.now()).encode( 'UTF-8')).hexdigest()[:4:] + ".tar.gz" ff = open(saved_name, 'wb') for salmon in resp: ff.write(salmon) ff.close() # Moves the file to reef and deletes the local copy requests.post('http://' + os.environ['Reef_IP'] + ':2001/reef/result_upload/' + os.environ['Reef_Key'] + '/' + UTOK, files={"file": open(saved_name, "rb")}) os.remove(saved_name) MESSAGE = Success_Message.replace( "DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n" MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n" MESSAGE += os.environ[ "SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join( BOCOM.split(' ')[1::]) MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK pp.send_mail(researcher_email, 'Succesful MIDAS build', MESSAGE) except Exception as e: print(e) r.delete(UTOK + ';' + MIDIR) # Deletes the unused container client.containers.prune() MESSAGE = Failure_Message.replace( "DATETIME", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK pp.send_mail(researcher_email, 'Failed MIDAS build', MESSAGE)
def complete_build(IMTAG, UTOK, MIDIR, DOCK_DOCK, BOCOM, FILES_PATH, boapp, job_id): researcher_email = pp.obtain_email(UTOK) # Updates job status mints.update_job_status(job_id, boapp, "Building image") try: user_image(IMTAG) # VolCon instructions # Deletes the image, submits the saved version to a mirror if boapp == "volcon": # Saves the image into a file img = image.get(IMTAG) resp = img.save() random_generated_dir = hashlib.sha256( str(datetime.datetime.now()).encode('UTF-8')).hexdigest()[:4:] image_dir = os.getcwd() + "/" + random_generated_dir os.mkdir(image_dir) full_image_path = image_dir + "/image.tar.gz" with open(full_image_path, "wb") as ff: for salmon in resp: ff.write(salmon) VolCon_ID = uuid.uuid4().hex mirror_IP = mirror.get_random_mirror() # Move image to the mirror mirror.upload_file_to_mirror(full_image_path, mirror_IP, VolCon_ID) # Moves the file to where it belongs saved_name = "image_" + hashlib.sha256( str(datetime.datetime.utcnow()).encode( 'UTF-8')).hexdigest()[:4:] + ".tar.gz" shutil.move(full_image_path, saved_name) # Move commands to mirror Commands = BOCOM # Add job to VolCon # Set as medium priority GPU_needed = 0 mints.make_MIDAS_job_available(job_id, "CUSTOM", Commands, GPU_needed, VolCon_ID, "Middle", public=0) mints.update_mirror_ip(VolCon_ID, mirror_IP) # MIDAS cannot accept GPU jobs job_info = { "Image": "Custom", "Command": Commands, "TACC": 0, "GPU": 0, "VolCon_ID": VolCon_ID, "public": 0, "key": mirror.mirror_key(mirror_IP) } requests.post( 'http://' + mirror_IP + ":7000/volcon/mirror/v2/api/public/receive_job_files", json=job_info) # Moves data to Reef requests.post('http://' + os.environ['Reef_IP'] + ':2001/reef/result_upload/' + os.environ['Reef_Key'] + '/' + UTOK, files={"file": open(saved_name, "rb")}) # Deletes local copy os.remove(saved_name) # Removes the image container.prune() image.remove(IMTAG, force=True) # Email user with dockerfile MESSAGE = Success_Message.replace("DATETIME", mints.timnow()) MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n" MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n" MESSAGE += os.environ[ "SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace( "../", "") MESSAGE += "\n\nDownload the image using the following command:\n" MESSAGE += "curl -O " + os.environ[ "SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace( "../", "") MESSAGE += "\nThen load the image (sudo permission may be required):" MESSAGE += "\ndocker load < " + saved_name.replace("../", "") MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):" MESSAGE += "\ndocker run -it IMAGE_ID bash" MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join( BOCOM.split(' ')[1::]) MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK ec.send_mail_complete(researcher_email, "Succesful MIDAS build", MESSAGE, []) return None # Updates the database so that the MIDAS job can be processed mints.make_boinc2docker_MIDAS_job_available(job_id, IMTAG.lower(), BOCOM) # Saves the docker image and sends the user the dockerfile and a link to the tar ball # docker-py documentation was erronous img = image.get(IMTAG) resp = img.save() # Creates a file, recycled everytime the program runs saved_name = "image." + hashlib.sha256( str(datetime.datetime.now()).encode( 'UTF-8')).hexdigest()[:4:] + ".tar.gz" ff = open(saved_name, 'wb') for salmon in resp: ff.write(salmon) ff.close() # Moves the file to reef and deletes the local copy requests.post('http://' + os.environ['Reef_IP'] + ':2001/reef/result_upload/' + os.environ['Reef_Key'] + '/' + UTOK, files={"file": open(saved_name, "rb")}) os.remove(saved_name) MESSAGE = Success_Message.replace("DATETIME", mints.timnow()) MESSAGE += "\n\nClick on the following link to obtain a compressed version of the application docker image.\n" MESSAGE += "You are welcome to upload the image on dockerhub in order to reduce the future job processing time for the same application (no allocation will be discounted): \n" MESSAGE += os.environ[ "SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name MESSAGE += "\n\nDownload the image using the following command:\n" MESSAGE += "curl -O " + os.environ[ "SERVER_IP"] + ":5060/boincserver/v2/reef/results/" + UTOK + "/" + saved_name.replace( "../", "") MESSAGE += "\nThen load the image (sudo permission may be required):" MESSAGE += "\ndocker load < " + saved_name.replace("../", "") MESSAGE += "\nThe image ID will appear, which can then be used to create a container (sudo permission may be required):" MESSAGE += "\ndocker run -it IMAGE_ID bash" MESSAGE += "\n\nRun the following command on the image: \n" + ' '.join( BOCOM.split(' ')[1::]) MESSAGE += "\n\nThis is the Dockerfile we used to process your job: \n\n" + DOCK_DOCK ec.send_mail_complete(researcher_email, "Succesful MIDAS build", MESSAGE, []) except Exception as e: print(e) # Updates status and notified time mints.update_job_status_notified(job_id, boapp, "Error creating MIDAS Dockerfile", notified_date_provided=False, processing_error=None) # Deletes the unused container client.containers.prune() MESSAGE = Failure_Message.replace("DATETIME", mints.timnow()) MESSAGE += "\n\nDockerfile created below: \n\n" + DOCK_DOCK ec.send_mail_complete(researcher_email, "Failed MIDAS build", MESSAGE, [])