def finalize_file_paths(files): circularity = [] # First off, fix the output dir if we have one; others # may depend on it. if not PipelineFile.output_dir: # there is no output_dir specified in the pipeline # create a default one in our current working directory PipelineFile.output_dir = PipelineFile('default_output', '.', files, is_dir=True, is_input=False, default_output=True) PipelineFile.output_dir.finalize_file(files, circularity) if PipelineFile.output_dir.is_input or not PipelineFile.output_dir.create: if not os.path.exists(PipelineFile.output_dir.path): raise civet_exceptions.MissingFile( "default output directory flagged as input must exist at pipeline submission time" ) else: utilities.make_sure_path_exists(PipelineFile.output_dir.path) for fid in files: files[fid].finalize_file(files, circularity)
def log_dir(self): if not self._log_dir: self._log_dir = os.path.join( PipelineFile.get_output_dir(), 'logs', datetime.datetime.now().strftime('%Y%m%d_%H%M%S')) utilities.make_sure_path_exists(self._log_dir) return self._log_dir
def __init__(self, log_dir="log", submit_with_hold=True, pbs_server=None, pipeline_bin=None, validation_cmd="ls -l", execution_log_dir=None, queue=None, submit=True, epilogue_email=None, pipeline_path=None): self.held_jobs = [] self.submit_with_hold = submit_with_hold self.validation_cmd = validation_cmd self.log_dir = os.path.abspath(log_dir) self._job_names = [] self._server = pbs_server self.pipeline_bin = pipeline_bin self.execution_log_dir = execution_log_dir self.queue = queue self.submit = submit self._id_seq = 0 # used to fake Torque job IDs when self.submit is False self.need_to_write_epilogue = True self.epilogue_email = epilogue_email self.pipeline_path = pipeline_path if self.execution_log_dir: self.execution_log_dir = os.path.abspath(self.execution_log_dir) self.epilogue_filename = os.path.join(self.log_dir, _SHELL_SCRIPT_DIR, "epilogue.sh") utilities.make_sure_path_exists(self.log_dir) self._id_log = open(os.path.join(log_dir, common.BATCH_ID_LOG), 'w') if not self.submit: # we aren't actually submitting jobs, create a file in the log # directory that civet_status can use to detect this case open(os.path.join(log_dir, common.NO_SUB_FLAG), 'w').close() # do some sanity checking on groups of files created in the log dir g_test_file = os.path.join(log_dir, ".group_test") open(g_test_file, 'w').close() gid = os.stat(g_test_file).st_gid os.remove(g_test_file) if gid not in os.getgroups(): print(textwrap.fill( "ERROR: The log directory {} has a group ID " "that you are not a member of. This will " "prevent Torque from operating correctly.".format( self.log_dir), width=80, break_long_words=False), file=sys.stderr) print("\nPIPELINE NOT SUBMITTED\n", file=sys.stderr) sys.exit(1)
def finalize_file(self, files, circularity): """ Take care of all the inter-file dependencies such as in_dir and based_on, as well as files passed in as parameters. """ if self.finalized: return # detect dependency cycles if self in circularity: msg = "File dependency cycle detected processing '{}' ".format( self.id) for f in circularity: msg = msg + "\n" + str(f) msg = msg + "\n\n" + str(self) raise civet_exceptions.ParseError(msg) circularity.append(self) self.parameter_to_path() self.apply_from_file(files, circularity) self.apply_based_on(files, circularity) if not self.is_string: # might raise civet_exception.ParseError # to be handled at a higher level self.apply_in_dir_and_create_temp(files, circularity) try: self.finalize_path() except Exception as e: # this was inserted to help diagnosing a programming error that # caused finalize_path() to fail in certain edge cases. # if it ever happens again this would provide useful information # the user could share with us to help locate the problem sys.exit("ERROR CALLING finalize_path() for {}: {}\n" "{}".format(self.id, self.path, e)) self.finalized = True # Make sure a directory exists, unless explicitly requested # to not do so. if self._is_dir and self.create: utilities.make_sure_path_exists(self.path) check = circularity.pop() if check != self: print("circularity.pop() failed!\ncheck:{}".format(check), file=sys.stderr) print(" self:{}".format(self), file=sys.stderr) sys.exit(1)
def apply_in_dir_and_create_temp(self, files, circularity): ind = self.in_dir if (not ind) and (not self.is_temp): return dir = PipelineFile.get_output_dir() utilities.make_sure_path_exists(dir) if ind: if ind not in files: msg = ("ERROR: while processing file with id: '{}', " "in_dir is unknown file: '{}'".format(self.id, ind)) raise civet_exceptions.ParseError(msg) indf = files[ind] indf.fix_up_file(files, circularity) dir = indf.path if self.is_list: return elif self.is_temp and not self.path: # If it is an anonymous temp, we'll create it in # the proper directory if self._is_dir: self.path = tempfile.mkdtemp(dir=dir) else: t = tempfile.NamedTemporaryFile(dir=dir, delete=False) name = t.name t.close() self.path = name if ind: self.in_dir = None elif ind: if os.path.isabs(self.path): raise civet_exceptions.ParseError( "Can't combine 'in_dir' attribute with absolute path") # Apply the containing directory to the path... self.path = os.path.join(dir, self.path) # in_dir has been applied, clear it. self.in_dir = None
def save_path(self): self.location = 'results/{}_{}_{}-{}/'.format(self.encoder.latent_dim, self.strategy, self.args.epochs, self.args.batch) utilities.make_sure_path_exists(self.location)
def fix_up_file(self, files, circularity): """ Take care of all the inter-file dependencies such as in_dir and based_on, as well as files passed in as parameters. """ import pipeline_parse as PL if self._is_fixed_up: return # detect dependency cycles if self in circularity: msg = "File dependency cycle detected processing '{}' ".format( self.id) for f in circularity: msg = msg + "\n" + str(f) msg = msg + "\n\n" + str(self) raise civet_exceptions.ParseError(msg) circularity.append(self) self.parameter_to_path() self.apply_from_file(files, circularity) self.apply_based_on(files, circularity) if self is PipelineFile.output_dir: if PL.directory_version == 2: stamp_dir = "{0}-{1}".format( datetime.datetime.now().strftime('%Y%m%d_%H%M%S'), os.getpid()) self.path = os.path.join(self.path, stamp_dir) utilities.make_sure_path_exists(self.path) else: # might raise civet_exception.ParseError, to be handled at a higher level self.apply_in_dir_and_create_temp(files, circularity) # Turn all the paths into an absolute path, so changes in # working directory throughout the pipeline lifetime don't # foul us up. First check if the file doesn't have a path at all # i.e., just a filename. If so, and it is not an input file, # place it in the output directory. if self.is_list: if self.in_dir: #filelist is comprised of a directory and pattern, #convert the directory to an absolute path self.in_dir = os.path.abspath(files[self.in_dir].path) elif self.list_from_param: # file list is passed as a parameter, might be comma delimited # convert paths in list to absolute path file_list = [] for f in self.path.split(','): file_list.append(os.path.abspath(f)) self.path = ','.join(file_list) elif not self.is_string: path = self.path if (os.path.split(path)[0] == '' and (not self.is_input) and self != PipelineFile.output_dir and (PipelineFile.output_dir is None or PipelineFile.output_dir._is_fixed_up)): path = os.path.join(PipelineFile.get_output_dir(), path) self.path = os.path.abspath(path) self._is_fixed_up = True # Make sure a directory exists, unless explicitly requested # to not do so. if self._is_dir and self.create: utilities.make_sure_path_exists(self.path) check = circularity.pop() if check != self: print("circularity.pop() failed!\ncheck:{}".format(check), file=sys.stderr) print(" self:{}".format(self), file=sys.stderr) sys.exit(1)