def dag_marker_filename(wuname):
"""
Takes a workunit name and returns a string path within
the dag_lists directory of the project. This is a fan out directory
tree similar to downloads/
@param wuname:String representation of the workunit name
@type wuname: str
@return: String filename for a dag marker (file not created).
@rtype: str
@raise dag.DagException: if dag_lists is not created
"""
from os import path as OP
import os
import dag
project_path = boinctools.project_path
cwd = os.getcwd()
if cwd != project_path:
os.chdir(project_path)
if not OP.isdir("dag_lists"):
raise dag.DagException("Missing dag_lists in project directory: '%s'\n"
"Current working directory: %s"
% (project_path, os.getcwd()))
os.chdir("dag_lists")
marker_filename = (boinctools.dir_hier_path(wuname)
.replace("download/", "dag_lists/"))
marker_dir = OP.dirname(marker_filename)
if not OP.isdir(marker_dir):
os.mkdir(marker_dir)
os.chmod(marker_dir, 0777)
if cwd != os.getcwd():
os.chdir(cwd)
return marker_filename
def clean(result):
import re
import dag.util as dag_utils
import boinctools
import dag, dag.boinc
import shutil
import stat
from os import path as OP
import os
if len(result.name) >= 2:
if result.name[-2:] != "_0":
print("Not cleaning %s" % result.name)
return True
print("Cleaning %s" % result.name)
wuname = re.findall(r"^(.*)_\d*$", result.name)
if len(wuname) == 0:
print("Malformed result name")
return None
wuname = wuname[0]
try:
the_dag = dag.boinc.result_to_dag(result.name)
except dag_utils.NoDagMarkerException as ndme:
print("Warning: Missing dag")
print("Skipping clean up")
print("Message:")
print(ndme.message)
return False
except dag.MissingDAGFile as mdf:
print(
"Missing dag file for result '%s'. Attempting to move output to invalid_results directory"
% result.name)
for output_file in result.output_files:
boinctools.save_bad_res_output(output_file[0], wuname)
return False
if not the_dag:
return False
dagpath = dag.boinc.marker_to_dagpath(
dag.boinc.dag_marker_filename(wuname))
dagdir = OP.split(dagpath)[0]
print("Getting process %s" % wuname)
proc = the_dag.get_process(wuname)
if not proc:
print(
"%s was not found in the job batch file %s. Moving resultfile to invalid_results"
% (wuname, dagpath))
for output_file in result.output_files:
dag.boinc.save_bad_res_output(output_file[0], wuname)
return False
if not proc.output_files:
return True
source_file = "%s_0" % result.name
output_file = proc.output_files[0]
logical_name = output_file.logical_name
print("Clean filename \"%s\"" % logical_name)
# Get output dir
if output_file.dir:
output_dir = proc.directory # final destination (If it can be written there)
else:
output_dir = dagdir
# If the result is valid, but the data in the bad_results
# directory.
if proc.state not in [dag.States.SUCCESS, dag.States.RUNNING]:
output_dir = OP.join(output_dir, "bad_results")
print(
"Process has not been marked as successful. It is %s instead. Saving output in %s"
% (dag.strstate(proc.state), output_dir))
if not OP.isdir(output_dir):
os.mkdir(output_dir)
dest_file = OP.join(output_dir, output_file.logical_name)
upload_path = boinctools.dir_hier_path(source_file).replace(
"/download", "/upload")
if not OP.isfile(upload_path):
print("Output file not found: '%s'" % upload_path)
return False
if not OP.isfile(dest_file):
# Copy file. If it does not exist, move it to the invalid_results directory
print("Copying {0} to {1}.".format(upload_path, dest_file))
try:
shutil.copy(upload_path, dest_file)
OP.os.chmod(
dest_file,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
except Exception as e:
dag.boinc.save_bad_res_output(upload_path, wuname)
print(
"ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s"
% (dagdir, wuname))
print("ERROR - Message:\n%s" % e.message)
if isinstance(e, IOError):
print(e.strerror)
raise e
else: #output already exists, append.
try:
with open(dest_file, "a") as old_file:
old_file.write("\n")
old_file.write(open(upload_path, "r").read())
except Exception as e:
dag.boinc.save_bad_res_output(upload_path, wuname)
print(
"ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s"
% (dagdir, wuname))
print("ERROR - Message:\n%s" % e.message)
if isinstance(e, IOError):
print(e.strerror)
raise e
return True
def clean(result):
import re
import dag.util as dag_utils
import boinctools
import dag,dag.boinc
import shutil
import stat
from os import path as OP
import os
if len(result.name) >= 2:
if result.name[-2:] != "_0":
print("Not cleaning %s" % result.name)
return True
print("Cleaning %s" % result.name)
wuname = re.findall(r"^(.*)_\d*$",result.name)
if len(wuname) == 0:
print("Malformed result name")
return None
wuname = wuname[0]
try:
the_dag = dag.boinc.result_to_dag(result.name)
except dag_utils.NoDagMarkerException as ndme:
print("Warning: Missing dag")
print("Skipping clean up" )
print("Message:")
print(ndme.message)
return False
except dag.MissingDAGFile as mdf:
print("Missing dag file for result '%s'. Attempting to move output to invalid_results directory" % result.name)
for output_file in result.output_files:
boinctools.save_bad_res_output(output_file[0],wuname)
return False
if not the_dag:
return False
dagpath = dag.boinc.marker_to_dagpath(dag.boinc.dag_marker_filename(wuname))
dagdir = OP.split(dagpath)[0]
print("Getting process %s" % wuname)
proc = the_dag.get_process(wuname)
if not proc:
print("%s was not found in the job batch file %s. Moving resultfile to invalid_results" % (wuname, dagpath))
for output_file in result.output_files:
dag.boinc.save_bad_res_output(output_file[0],wuname)
return False
if not proc.output_files:
return True
source_file = "%s_0" % result.name
output_file = proc.output_files[0]
logical_name = output_file.logical_name
print("Clean filename \"%s\"" % logical_name)
# Get output dir
if output_file.dir:
output_dir = proc.directory # final destination (If it can be written there)
else:
output_dir = dagdir
# If the result is valid, but the data in the bad_results
# directory.
if proc.state not in [dag.States.SUCCESS,dag.States.RUNNING]:
output_dir = OP.join(output_dir,"bad_results")
print("Process has not been marked as successful. It is %s instead. Saving output in %s" % (dag.strstate(proc.state), output_dir))
if not OP.isdir(output_dir):
os.mkdir(output_dir)
dest_file = OP.join(output_dir,output_file.logical_name)
upload_path = boinctools.dir_hier_path(source_file).replace("/download","/upload")
if not OP.isfile(upload_path):
print("Output file not found: '%s'" % upload_path)
return False
if not OP.isfile(dest_file):
# Copy file. If it does not exist, move it to the invalid_results directory
print("Copying {0} to {1}.".format(upload_path,dest_file))
try:
shutil.copy(upload_path,dest_file)
OP.os.chmod(dest_file,stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
except Exception as e:
dag.boinc.save_bad_res_output(upload_path,wuname)
print("ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir,wuname))
print("ERROR - Message:\n%s" % e.message)
if isinstance(e,IOError):
print(e.strerror)
raise e
else: #output already exists, append.
try:
with open(dest_file,"a") as old_file:
old_file.write("\n")
old_file.write(open(upload_path,"r").read())
except Exception as e:
dag.boinc.save_bad_res_output(upload_path,wuname)
print("ERROR - Could not copy result output file to data directory, %s. It was copied to invalid_results/%s" % (dagdir,wuname))
print("ERROR - Message:\n%s" % e.message)
if isinstance(e,IOError):
print(e.strerror)
raise e
return True
