def check_output(self, job, wait=3):
""" Raise exception if output files of job are missing. """
for f in job.expanded_output:
if not f.exists:
logger.warning(
"Output file {} not present. Waiting {} "
"seconds to ensure that this is not because of filesystem "
"latency.".format(f, wait))
while not f.exists and wait > 0:
wait -= 1
time.sleep(1)
if not f.exists:
raise MissingOutputException("Output file {} not "
"produced by rule {}.".format(
f, job.rule.name),
lineno=job.rule.lineno,
snakefile=job.rule.snakefile)
input_maxtime = job.input_maxtime
if input_maxtime is not None:
output_mintime = job.output_mintime
if output_mintime is not None and output_mintime < input_maxtime:
raise RuleException(
"Output files {} are older than input "
"files. Did you extract an archive? Make sure that output "
"files have a more recent modification date than the "
"archive, e.g. by using 'touch'.".format(", ".join(
job.expanded_output)),
rule=job.rule)
def touch(self):
try:
os.utime(self.file, None)
except OSError as e:
if e.errno == 2:
raise MissingOutputException(
"Output file {} of rule {} shall be touched but "
"does not exist.".format(self.file, self.rule.name),
lineno=self.rule.lineno,
snakefile=self.rule.snakefile)
else:
raise e
def touch(self, times=None):
""" times must be 2-tuple: (atime, mtime) """
try:
lutime(self.file, times)
except OSError as e:
if e.errno == 2:
raise MissingOutputException(
"Output file {} of rule {} shall be touched but "
"does not exist.".format(self.file, self.rule.name),
lineno=self.rule.lineno,
snakefile=self.rule.snakefile)
else:
raise e
def check_and_touch_output(self, job, wait=3):
""" Raise exception if output files of job are missing. """
expanded_output = [job.shadowed_path(path) for path in job.expanded_output]
try:
wait_for_files(expanded_output, latency_wait=wait)
except IOError as e:
raise MissingOutputException(str(e), rule=job.rule)
#It is possible, due to archive expansion or cluster clock skew, that
#the files appear older than the input. But we know they must be new,
#so touch them to update timestamps.
#Note that if the input files somehow have a future date then this will
#not currently be spotted and the job will always be re-run.
#Also, don't touch directories, as we can't guarantee they were removed.
for f in expanded_output:
if not os.path.isdir(f):
f.touch()
def check_output(self, job, wait=3):
""" Raise exception if output files of job are missing. """
try:
wait_for_files(job.expanded_output, latency_wait=wait)
except IOError as e:
raise MissingOutputException(str(e), rule=job.rule)
input_maxtime = job.input_maxtime
if input_maxtime is not None:
output_mintime = job.output_mintime
if output_mintime is not None and output_mintime < input_maxtime:
raise RuleException(
"Output files {} are older than input "
"files. Did you extract an archive? Make sure that output "
"files have a more recent modification date than the "
"archive, e.g. by using 'touch'.".format(", ".join(
job.expanded_output)),
rule=job.rule)
def touch(self, times=None):
""" times must be 2-tuple: (atime, mtime) """
try:
if self.is_directory:
file = os.path.join(self.file, ".snakemake_timestamp")
# Create the flag file if it doesn't exist
if not os.path.exists(file):
with open(file, "w") as f:
pass
lutime(file, times)
else:
lutime(self.file, times)
except OSError as e:
if e.errno == 2:
raise MissingOutputException(
"Output file {} of rule {} shall be touched but "
"does not exist.".format(self.file, self.rule.name),
lineno=self.rule.lineno,
snakefile=self.rule.snakefile)
else:
raise e