def write_annotation_file(self, path):
'''
Write the YAML annotation after a successful or failed run. The
annotation can later be used to render the process graph.
'''
# now write the annotation
log = {}
log['pid'] = os.getpid()
log['step'] = {}
log['step']['options'] = self.get_step().get_options()
log['step']['name'] = self.get_step().get_step_name()
log['step']['cores'] = self.get_step()._cores
log['run'] = {}
log['run']['run_info'] = self.as_dict()
log['run']['run_id'] = self.get_run_id()
log['run']['temp_directory'] = self.get_temp_output_directory()
# if a submit script was used ...
if os.path.exists(self.get_submit_script_file()):
# ... read it and store it ...
with open(self.get_submit_script_file(), 'r') as f:
log['run']['submit_script'] = f.read()
# ... finally delete it
os.unlink(self.get_submit_script_file())
log['run']['known_paths'] = self.get_known_paths()
log['config'] = self.get_step().get_pipeline().config
log['git_hash_tag'] = self.get_step().get_pipeline().git_hash_tag
log['tool_versions'] = {}
for tool in self.get_step()._tools.keys():
log['tool_versions'][tool] = self.get_step().get_pipeline()\
.tool_versions[tool]
log['pipeline_log'] = self.get_step()._pipeline_log
log['start_time'] = self.get_step().start_time
log['end_time'] = self.get_step().end_time
if self.get_step().get_pipeline().git_dirty_diff:
log['git_dirty_diff'] = self.get_step().get_pipeline(
).git_dirty_diff
if self.get_step().get_pipeline().caught_signal is not None:
log['signal'] = self.get_step().get_pipeline().caught_signal
annotation_yaml = yaml.dump(log, default_flow_style=False)
annotation_path = os.path.join(
path, ".%s-annotation-%s.yaml" %
(self.get_run_id(), misc.str_to_sha1_b62(annotation_yaml)[:6]))
# overwrite the annotation if it already exists
with open(annotation_path, 'w') as f:
f.write(annotation_yaml)
return annotation_path, annotation_yaml
def write_annotation_file(self, path):
'''
Write the YAML annotation after a successful or failed run. The
annotation can later be used to render the process graph.
'''
# now write the annotation
log = {}
log['pid'] = os.getpid()
log['step'] = {}
log['step']['options'] = self.get_step().get_options()
log['step']['name'] = self.get_step().get_step_name()
log['step']['cores'] = self.get_step()._cores
log['run'] = {}
log['run']['run_info'] = self.as_dict()
log['run']['run_id'] = self.get_run_id()
log['run']['temp_directory'] = self.get_temp_output_directory()
# if a submit script was used ...
if os.path.exists(self.get_submit_script_file()):
# ... read it and store it ...
with open(self.get_submit_script_file(), 'r') as f:
log['run']['submit_script'] = f.read()
# ... finally delete it
os.unlink(self.get_submit_script_file())
log['run']['known_paths'] = self.get_known_paths()
log['config'] = self.get_step().get_pipeline().config
log['git_hash_tag'] = self.get_step().get_pipeline().git_hash_tag
log['tool_versions'] = {}
for tool in self.get_step()._tools.keys():
log['tool_versions'][tool] = self.get_step().get_pipeline()\
.tool_versions[tool]
log['pipeline_log'] = self.get_step()._pipeline_log
log['start_time'] = self.get_step().start_time
log['end_time'] = self.get_step().end_time
if self.get_step().get_pipeline().git_dirty_diff:
log['git_dirty_diff'] = self.get_step().get_pipeline().git_dirty_diff
if self.get_step().get_pipeline().caught_signal is not None:
log['signal'] = self.get_step().get_pipeline().caught_signal
annotation_yaml = yaml.dump(log, default_flow_style = False)
annotation_path = os.path.join(
path, ".%s-annotation-%s.yaml" %
(self.get_run_id(), misc.str_to_sha1_b62(annotation_yaml)[:6]))
# overwrite the annotation if it already exists
with open(annotation_path, 'w') as f:
f.write(annotation_yaml)
return annotation_path, annotation_yaml
def add_temporary_file(self, prefix='', suffix='', designation=None):
'''
Returns the name of a temporary file (created by tempfile library).
Name and output directory placeholder are concatenated. The concatenated
string is returned and stored in a list. The placeholder is immediately
properly adjusted by @replace_output_dir_du_jour.
'''
count = len(self._temp_paths)
temp_placeholder = str()
while True:
hashtag = misc.str_to_sha1_b62('%s.%s.%s' %
(prefix, count, suffix))
temp_name = prefix + hashtag + suffix
temp_placeholder = os.path.join(
self.get_output_directory_du_jour_placeholder(), temp_name)
if not temp_placeholder in self._temp_paths:
break
else:
count += 1
logger.info("Temporary file (#%s): %s" %
(len(self._temp_paths) + 1, temp_name))
# _known_paths dict is logged
known_paths = dict()
known_paths[temp_placeholder] = {
'label': os.path.basename(temp_placeholder),
'designation': designation,
'type': ''
}
self.add_known_paths(known_paths)
# _temp_paths list contains all temporary files which are going to be
# deleted
self._temp_paths.append(temp_placeholder)
return temp_placeholder
def get_execution_hashtag(self):
'''
Creates a hash tag based on the commands to be executed.
This causes runs to be marked for rerunning if the commands to be
executed change.
'''
# Store step state
previous_state = self.get_step()._state
# Set step state to DECLARING to avoid circular dependencies
self.get_step()._state = abst.AbstractStep.states.DECLARING
cmd_by_eg = dict()
eg_count = 0
for exec_group in self.get_exec_groups():
eg_count += 1
cmd_by_eg[eg_count] = dict()
pipe_count, cmd_count = (0, 0)
for poc in exec_group.get_pipes_and_commands():
# for each pipe or command (poc)
# check if it is a pipeline ...
if isinstance(poc, pipeline_info.PipelineInfo):
pipe_count += 1
cmd_by_eg[eg_count]['Pipe %s' % pipe_count] = list()
for command in poc.get_commands():
cmd_by_eg[eg_count]['Pipe %s' % pipe_count].append(
command.get_command())
# ... or a command
elif isinstance(poc, command_info.CommandInfo):
cmd_count += 1
cmd_by_eg[eg_count]['Cmd %s' %
cmd_count] = poc.get_command()
# Set step state back to original state
self.get_step()._state = previous_state
return misc.str_to_sha1_b62(json.dumps(cmd_by_eg))[0:8]
def add_temporary_file(self, prefix = '', suffix = '', designation = None):
'''
Returns the name of a temporary file (created by tempfile library).
Name and output directory placeholder are concatenated. The concatenated
string is returned and stored in a list. The placeholder is immediately
properly adjusted by @replace_output_dir_du_jour.
'''
count = len(self._temp_paths)
temp_placeholder = str()
while True:
hashtag = misc.str_to_sha1_b62('%s.%s.%s' % (prefix, count, suffix))
temp_name = prefix + hashtag + suffix
temp_placeholder = os.path.join(
self.get_output_directory_du_jour_placeholder(), temp_name)
if not temp_placeholder in self._temp_paths:
break
else:
count += 1
logger.info("Temporary file (#%s): %s" %
(len(self._temp_paths) + 1, temp_name) )
# _known_paths dict is logged
known_paths = dict()
known_paths[temp_placeholder] = {
'label': os.path.basename(temp_placeholder),
'designation': designation,
'type': ''
}
self.add_known_paths(known_paths)
# _temp_paths list contains all temporary files which are going to be
# deleted
self._temp_paths.append(temp_placeholder)
return temp_placeholder
def get_execution_hashtag(self):
'''
Creates a hash tag based on the commands to be executed.
This causes runs to be marked for rerunning if the commands to be
executed change.
'''
# Store step state
previous_state = self.get_step()._state
# Set step state to DECLARING to avoid circular dependencies
self.get_step()._state = abst.AbstractStep.states.DECLARING
cmd_by_eg = dict()
eg_count = 0
for exec_group in self.get_exec_groups():
eg_count += 1
cmd_by_eg[eg_count] = dict()
pipe_count, cmd_count = (0, 0)
for poc in exec_group.get_pipes_and_commands():
# for each pipe or command (poc)
# check if it is a pipeline ...
if isinstance(poc, pipeline_info.PipelineInfo):
pipe_count += 1
cmd_by_eg[eg_count]['Pipe %s' % pipe_count] = list()
for command in poc.get_commands():
cmd_by_eg[eg_count]['Pipe %s' % pipe_count].append(
command.get_command())
# ... or a command
elif isinstance(poc, command_info.CommandInfo):
cmd_count += 1
cmd_by_eg[eg_count]['Cmd %s' % cmd_count] = poc.get_command()
# Set step state back to original state
self.get_step()._state = previous_state
return misc.str_to_sha1_b62(json.dumps(cmd_by_eg))[0:8]