def finalize_file_paths(files):
circularity = []
# First off, fix the output dir if we have one; others
# may depend on it.
if not PipelineFile.output_dir:
# there is no output_dir specified in the pipeline
# create a default one in our current working directory
PipelineFile.output_dir = PipelineFile('default_output',
'.',
files,
is_dir=True,
is_input=False,
default_output=True)
PipelineFile.output_dir.finalize_file(files, circularity)
if PipelineFile.output_dir.is_input or not PipelineFile.output_dir.create:
if not os.path.exists(PipelineFile.output_dir.path):
raise civet_exceptions.MissingFile(
"default output directory flagged as input must exist at pipeline submission time"
)
else:
utilities.make_sure_path_exists(PipelineFile.output_dir.path)
for fid in files:
files[fid].finalize_file(files, circularity)
def log_dir(self):
if not self._log_dir:
self._log_dir = os.path.join(
PipelineFile.get_output_dir(), 'logs',
datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
utilities.make_sure_path_exists(self._log_dir)
return self._log_dir
def __init__(self,
log_dir="log",
submit_with_hold=True,
pbs_server=None,
pipeline_bin=None,
validation_cmd="ls -l",
execution_log_dir=None,
queue=None,
submit=True,
epilogue_email=None,
pipeline_path=None):
self.held_jobs = []
self.submit_with_hold = submit_with_hold
self.validation_cmd = validation_cmd
self.log_dir = os.path.abspath(log_dir)
self._job_names = []
self._server = pbs_server
self.pipeline_bin = pipeline_bin
self.execution_log_dir = execution_log_dir
self.queue = queue
self.submit = submit
self._id_seq = 0 # used to fake Torque job IDs when self.submit is False
self.need_to_write_epilogue = True
self.epilogue_email = epilogue_email
self.pipeline_path = pipeline_path
if self.execution_log_dir:
self.execution_log_dir = os.path.abspath(self.execution_log_dir)
self.epilogue_filename = os.path.join(self.log_dir, _SHELL_SCRIPT_DIR,
"epilogue.sh")
utilities.make_sure_path_exists(self.log_dir)
self._id_log = open(os.path.join(log_dir, common.BATCH_ID_LOG), 'w')
if not self.submit:
# we aren't actually submitting jobs, create a file in the log
# directory that civet_status can use to detect this case
open(os.path.join(log_dir, common.NO_SUB_FLAG), 'w').close()
# do some sanity checking on groups of files created in the log dir
g_test_file = os.path.join(log_dir, ".group_test")
open(g_test_file, 'w').close()
gid = os.stat(g_test_file).st_gid
os.remove(g_test_file)
if gid not in os.getgroups():
print(textwrap.fill(
"ERROR: The log directory {} has a group ID "
"that you are not a member of. This will "
"prevent Torque from operating correctly.".format(
self.log_dir),
width=80,
break_long_words=False),
file=sys.stderr)
print("\nPIPELINE NOT SUBMITTED\n", file=sys.stderr)
sys.exit(1)
def finalize_file(self, files, circularity):
"""
Take care of all the inter-file dependencies such as
in_dir and based_on, as well as files passed in as
parameters.
"""
if self.finalized:
return
# detect dependency cycles
if self in circularity:
msg = "File dependency cycle detected processing '{}' ".format(
self.id)
for f in circularity:
msg = msg + "\n" + str(f)
msg = msg + "\n\n" + str(self)
raise civet_exceptions.ParseError(msg)
circularity.append(self)
self.parameter_to_path()
self.apply_from_file(files, circularity)
self.apply_based_on(files, circularity)
if not self.is_string:
# might raise civet_exception.ParseError
# to be handled at a higher level
self.apply_in_dir_and_create_temp(files, circularity)
try:
self.finalize_path()
except Exception as e:
# this was inserted to help diagnosing a programming error that
# caused finalize_path() to fail in certain edge cases.
# if it ever happens again this would provide useful information
# the user could share with us to help locate the problem
sys.exit("ERROR CALLING finalize_path() for {}: {}\n"
"{}".format(self.id, self.path, e))
self.finalized = True
# Make sure a directory exists, unless explicitly requested
# to not do so.
if self._is_dir and self.create:
utilities.make_sure_path_exists(self.path)
check = circularity.pop()
if check != self:
print("circularity.pop() failed!\ncheck:{}".format(check),
file=sys.stderr)
print(" self:{}".format(self), file=sys.stderr)
sys.exit(1)
def apply_in_dir_and_create_temp(self, files, circularity):
ind = self.in_dir
if (not ind) and (not self.is_temp):
return
dir = PipelineFile.get_output_dir()
utilities.make_sure_path_exists(dir)
if ind:
if ind not in files:
msg = ("ERROR: while processing file with id: '{}', "
"in_dir is unknown file: '{}'".format(self.id, ind))
raise civet_exceptions.ParseError(msg)
indf = files[ind]
indf.fix_up_file(files, circularity)
dir = indf.path
if self.is_list:
return
elif self.is_temp and not self.path:
# If it is an anonymous temp, we'll create it in
# the proper directory
if self._is_dir:
self.path = tempfile.mkdtemp(dir=dir)
else:
t = tempfile.NamedTemporaryFile(dir=dir, delete=False)
name = t.name
t.close()
self.path = name
if ind:
self.in_dir = None
elif ind:
if os.path.isabs(self.path):
raise civet_exceptions.ParseError(
"Can't combine 'in_dir' attribute with absolute path")
# Apply the containing directory to the path...
self.path = os.path.join(dir, self.path)
# in_dir has been applied, clear it.
self.in_dir = None
def save_path(self):
self.location = 'results/{}_{}_{}-{}/'.format(self.encoder.latent_dim, self.strategy, self.args.epochs, self.args.batch)
utilities.make_sure_path_exists(self.location)
def fix_up_file(self, files, circularity):
"""
Take care of all the inter-file dependencies such as
in_dir and based_on, as well as files passed in as
parameters.
"""
import pipeline_parse as PL
if self._is_fixed_up:
return
# detect dependency cycles
if self in circularity:
msg = "File dependency cycle detected processing '{}' ".format(
self.id)
for f in circularity:
msg = msg + "\n" + str(f)
msg = msg + "\n\n" + str(self)
raise civet_exceptions.ParseError(msg)
circularity.append(self)
self.parameter_to_path()
self.apply_from_file(files, circularity)
self.apply_based_on(files, circularity)
if self is PipelineFile.output_dir:
if PL.directory_version == 2:
stamp_dir = "{0}-{1}".format(
datetime.datetime.now().strftime('%Y%m%d_%H%M%S'),
os.getpid())
self.path = os.path.join(self.path, stamp_dir)
utilities.make_sure_path_exists(self.path)
else:
# might raise civet_exception.ParseError, to be handled at a higher level
self.apply_in_dir_and_create_temp(files, circularity)
# Turn all the paths into an absolute path, so changes in
# working directory throughout the pipeline lifetime don't
# foul us up. First check if the file doesn't have a path at all
# i.e., just a filename. If so, and it is not an input file,
# place it in the output directory.
if self.is_list:
if self.in_dir:
#filelist is comprised of a directory and pattern,
#convert the directory to an absolute path
self.in_dir = os.path.abspath(files[self.in_dir].path)
elif self.list_from_param:
# file list is passed as a parameter, might be comma delimited
# convert paths in list to absolute path
file_list = []
for f in self.path.split(','):
file_list.append(os.path.abspath(f))
self.path = ','.join(file_list)
elif not self.is_string:
path = self.path
if (os.path.split(path)[0] == '' and (not self.is_input)
and self != PipelineFile.output_dir
and (PipelineFile.output_dir is None
or PipelineFile.output_dir._is_fixed_up)):
path = os.path.join(PipelineFile.get_output_dir(), path)
self.path = os.path.abspath(path)
self._is_fixed_up = True
# Make sure a directory exists, unless explicitly requested
# to not do so.
if self._is_dir and self.create:
utilities.make_sure_path_exists(self.path)
check = circularity.pop()
if check != self:
print("circularity.pop() failed!\ncheck:{}".format(check),
file=sys.stderr)
print(" self:{}".format(self), file=sys.stderr)
sys.exit(1)