def dag_marker_filename(wuname): """ Takes a workunit name and returns a string path within the dag_lists directory of the project. This is a fan out directory tree similar to downloads/ @param wuname:String representation of the workunit name @type wuname: str @return: String filename for a dag marker (file not created). @rtype: str @raise dag.DagException: if dag_lists is not created """ from os import path as OP import os import dag project_path = boinctools.project_path cwd = os.getcwd() if cwd != project_path: os.chdir(project_path) if not OP.isdir("dag_lists"): raise dag.DagException("Missing dag_lists in project directory: '%s'\n" "Current working directory: %s" % (project_path, os.getcwd())) os.chdir("dag_lists") marker_filename = (boinctools.dir_hier_path(wuname) .replace("download/", "dag_lists/")) marker_dir = OP.dirname(marker_filename) if not OP.isdir(marker_dir): os.mkdir(marker_dir) os.chmod(marker_dir, 0777) if cwd != os.getcwd(): os.chdir(cwd) return marker_filename
def clean(result): import re import dag.util as dag_utils import boinctools import dag, dag.boinc import shutil import stat from os import path as OP import os if len(result.name) >= 2: if result.name[-2:] != "_0": print("Not cleaning %s" % result.name) return True print("Cleaning %s" % result.name) wuname = re.findall(r"^(.*)_\d*$", result.name) if len(wuname) == 0: print("Malformed result name") return None wuname = wuname[0] try: the_dag = dag.boinc.result_to_dag(result.name) except dag_utils.NoDagMarkerException as ndme: print("Warning: Missing dag") print("Skipping clean up") print("Message:") print(ndme.message) return False except dag.MissingDAGFile as mdf: print( "Missing dag file for result '%s'. Attempting to move output to invalid_results directory" % result.name) for output_file in result.output_files: boinctools.save_bad_res_output(output_file[0], wuname) return False if not the_dag: return False dagpath = dag.boinc.marker_to_dagpath( dag.boinc.dag_marker_filename(wuname)) dagdir = OP.split(dagpath)[0] print("Getting process %s" % wuname) proc = the_dag.get_process(wuname) if not proc: print( "%s was not found in the job batch file %s. Moving resultfile to invalid_results" % (wuname, dagpath)) for output_file in result.output_files: dag.boinc.save_bad_res_output(output_file[0], wuname) return False if not proc.output_files: return True source_file = "%s_0" % result.name output_file = proc.output_files[0] logical_name = output_file.logical_name print("Clean filename \"%s\"" % logical_name) # Get output dir if output_file.dir: output_dir = proc.directory # final destination (If it can be written there) else: output_dir = dagdir # If the result is valid, but the data in the bad_results # directory. if proc.state not in [dag.States.SUCCESS, dag.States.RUNNING]: output_dir = OP.join(output_dir, "bad_results") print( "Process has not been marked as successful. It is %s instead. Saving output in %s" % (dag.strstate(proc.state), output_dir)) if not OP.isdir(output_dir): os.mkdir(output_dir) dest_file = OP.join(output_dir, output_file.logical_name) upload_path = boinctools.dir_hier_path(source_file).replace( "/download", "/upload") if not OP.isfile(upload_path): print("Output file not found: '%s'" % upload_path) return False if not OP.isfile(dest_file): # Copy file. If it does not exist, move it to the invalid_results directory print("Copying {0} to {1}.".format(upload_path, dest_file)) try: shutil.copy(upload_path, dest_file) OP.os.chmod( dest_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP) except Exception as e: dag.boinc.save_bad_res_output(upload_path, wuname) print( "ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir, wuname)) print("ERROR - Message:\n%s" % e.message) if isinstance(e, IOError): print(e.strerror) raise e else: #output already exists, append. try: with open(dest_file, "a") as old_file: old_file.write("\n") old_file.write(open(upload_path, "r").read()) except Exception as e: dag.boinc.save_bad_res_output(upload_path, wuname) print( "ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir, wuname)) print("ERROR - Message:\n%s" % e.message) if isinstance(e, IOError): print(e.strerror) raise e return True
def clean(result): import re import dag.util as dag_utils import boinctools import dag,dag.boinc import shutil import stat from os import path as OP import os if len(result.name) >= 2: if result.name[-2:] != "_0": print("Not cleaning %s" % result.name) return True print("Cleaning %s" % result.name) wuname = re.findall(r"^(.*)_\d*$",result.name) if len(wuname) == 0: print("Malformed result name") return None wuname = wuname[0] try: the_dag = dag.boinc.result_to_dag(result.name) except dag_utils.NoDagMarkerException as ndme: print("Warning: Missing dag") print("Skipping clean up" ) print("Message:") print(ndme.message) return False except dag.MissingDAGFile as mdf: print("Missing dag file for result '%s'. Attempting to move output to invalid_results directory" % result.name) for output_file in result.output_files: boinctools.save_bad_res_output(output_file[0],wuname) return False if not the_dag: return False dagpath = dag.boinc.marker_to_dagpath(dag.boinc.dag_marker_filename(wuname)) dagdir = OP.split(dagpath)[0] print("Getting process %s" % wuname) proc = the_dag.get_process(wuname) if not proc: print("%s was not found in the job batch file %s. Moving resultfile to invalid_results" % (wuname, dagpath)) for output_file in result.output_files: dag.boinc.save_bad_res_output(output_file[0],wuname) return False if not proc.output_files: return True source_file = "%s_0" % result.name output_file = proc.output_files[0] logical_name = output_file.logical_name print("Clean filename \"%s\"" % logical_name) # Get output dir if output_file.dir: output_dir = proc.directory # final destination (If it can be written there) else: output_dir = dagdir # If the result is valid, but the data in the bad_results # directory. if proc.state not in [dag.States.SUCCESS,dag.States.RUNNING]: output_dir = OP.join(output_dir,"bad_results") print("Process has not been marked as successful. It is %s instead. Saving output in %s" % (dag.strstate(proc.state), output_dir)) if not OP.isdir(output_dir): os.mkdir(output_dir) dest_file = OP.join(output_dir,output_file.logical_name) upload_path = boinctools.dir_hier_path(source_file).replace("/download","/upload") if not OP.isfile(upload_path): print("Output file not found: '%s'" % upload_path) return False if not OP.isfile(dest_file): # Copy file. If it does not exist, move it to the invalid_results directory print("Copying {0} to {1}.".format(upload_path,dest_file)) try: shutil.copy(upload_path,dest_file) OP.os.chmod(dest_file,stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP) except Exception as e: dag.boinc.save_bad_res_output(upload_path,wuname) print("ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir,wuname)) print("ERROR - Message:\n%s" % e.message) if isinstance(e,IOError): print(e.strerror) raise e else: #output already exists, append. try: with open(dest_file,"a") as old_file: old_file.write("\n") old_file.write(open(upload_path,"r").read()) except Exception as e: dag.boinc.save_bad_res_output(upload_path,wuname) print("ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir,wuname)) print("ERROR - Message:\n%s" % e.message) if isinstance(e,IOError): print(e.strerror) raise e return True