示例#1
0
    def write_annotation_file(self, path):
        '''
        Write the YAML annotation after a successful or failed run. The
        annotation can later be used to render the process graph.
        '''

        # now write the annotation
        log = {}
        log['pid'] = os.getpid()
        log['step'] = {}
        log['step']['options'] = self.get_step().get_options()
        log['step']['name'] = self.get_step().get_step_name()
        log['step']['cores'] = self.get_step()._cores
        log['run'] = {}
        log['run']['run_info'] = self.as_dict()
        log['run']['run_id'] = self.get_run_id()
        log['run']['temp_directory'] = self.get_temp_output_directory()
        # if a submit script was used ...
        if os.path.exists(self.get_submit_script_file()):
            # ... read it and store it ...
            with open(self.get_submit_script_file(), 'r') as f:
                log['run']['submit_script'] = f.read()
            # ... finally delete it
            os.unlink(self.get_submit_script_file())
        log['run']['known_paths'] = self.get_known_paths()
        log['config'] = self.get_step().get_pipeline().config
        log['git_hash_tag'] = self.get_step().get_pipeline().git_hash_tag
        log['tool_versions'] = {}
        for tool in self.get_step()._tools.keys():
            log['tool_versions'][tool] = self.get_step().get_pipeline()\
                                                        .tool_versions[tool]
        log['pipeline_log'] = self.get_step()._pipeline_log
        log['start_time'] = self.get_step().start_time
        log['end_time'] = self.get_step().end_time
        if self.get_step().get_pipeline().git_dirty_diff:
            log['git_dirty_diff'] = self.get_step().get_pipeline(
            ).git_dirty_diff
        if self.get_step().get_pipeline().caught_signal is not None:
            log['signal'] = self.get_step().get_pipeline().caught_signal

        annotation_yaml = yaml.dump(log, default_flow_style=False)
        annotation_path = os.path.join(
            path, ".%s-annotation-%s.yaml" %
            (self.get_run_id(), misc.str_to_sha1_b62(annotation_yaml)[:6]))

        # overwrite the annotation if it already exists
        with open(annotation_path, 'w') as f:
            f.write(annotation_yaml)

        return annotation_path, annotation_yaml
示例#2
0
    def write_annotation_file(self, path):
        '''
        Write the YAML annotation after a successful or failed run. The
        annotation can later be used to render the process graph.
        '''
        
        # now write the annotation
        log = {}
        log['pid'] = os.getpid()
        log['step'] = {}
        log['step']['options'] = self.get_step().get_options()
        log['step']['name'] = self.get_step().get_step_name()
        log['step']['cores'] = self.get_step()._cores
        log['run'] = {}
        log['run']['run_info'] = self.as_dict()
        log['run']['run_id'] = self.get_run_id()
        log['run']['temp_directory'] = self.get_temp_output_directory()
        # if a submit script was used ...
        if os.path.exists(self.get_submit_script_file()):
            # ... read it and store it ...
            with open(self.get_submit_script_file(), 'r') as f:
                log['run']['submit_script'] = f.read()
            # ... finally delete it
            os.unlink(self.get_submit_script_file())
        log['run']['known_paths'] = self.get_known_paths()
        log['config'] = self.get_step().get_pipeline().config
        log['git_hash_tag'] = self.get_step().get_pipeline().git_hash_tag
        log['tool_versions'] = {}
        for tool in self.get_step()._tools.keys():
            log['tool_versions'][tool] = self.get_step().get_pipeline()\
                                                        .tool_versions[tool]
        log['pipeline_log'] = self.get_step()._pipeline_log
        log['start_time'] = self.get_step().start_time
        log['end_time'] = self.get_step().end_time
        if self.get_step().get_pipeline().git_dirty_diff:
            log['git_dirty_diff'] = self.get_step().get_pipeline().git_dirty_diff
        if self.get_step().get_pipeline().caught_signal is not None:
            log['signal'] = self.get_step().get_pipeline().caught_signal

        annotation_yaml = yaml.dump(log, default_flow_style = False)
        annotation_path = os.path.join(
            path, ".%s-annotation-%s.yaml" % 
            (self.get_run_id(), misc.str_to_sha1_b62(annotation_yaml)[:6]))

        # overwrite the annotation if it already exists
        with open(annotation_path, 'w') as f:
            f.write(annotation_yaml)
            
        return annotation_path, annotation_yaml
示例#3
0
    def add_temporary_file(self, prefix='', suffix='', designation=None):
        '''
        Returns the name of a temporary file (created by tempfile library).
        Name and output directory placeholder are concatenated. The concatenated
        string is returned and stored in a list. The placeholder is immediately
        properly adjusted by @replace_output_dir_du_jour.
        '''
        count = len(self._temp_paths)
        temp_placeholder = str()

        while True:
            hashtag = misc.str_to_sha1_b62('%s.%s.%s' %
                                           (prefix, count, suffix))
            temp_name = prefix + hashtag + suffix
            temp_placeholder = os.path.join(
                self.get_output_directory_du_jour_placeholder(), temp_name)

            if not temp_placeholder in self._temp_paths:
                break
            else:
                count += 1

        logger.info("Temporary file (#%s): %s" %
                    (len(self._temp_paths) + 1, temp_name))

        # _known_paths dict is logged
        known_paths = dict()
        known_paths[temp_placeholder] = {
            'label': os.path.basename(temp_placeholder),
            'designation': designation,
            'type': ''
        }
        self.add_known_paths(known_paths)
        # _temp_paths list contains all temporary files which are going to be
        # deleted
        self._temp_paths.append(temp_placeholder)
        return temp_placeholder
示例#4
0
    def get_execution_hashtag(self):
        '''
        Creates a hash tag based on the commands to be executed.

        This causes runs to be marked for rerunning if the commands to be
        executed change.
        '''

        # Store step state
        previous_state = self.get_step()._state
        # Set step state to DECLARING to avoid circular dependencies
        self.get_step()._state = abst.AbstractStep.states.DECLARING

        cmd_by_eg = dict()
        eg_count = 0
        for exec_group in self.get_exec_groups():
            eg_count += 1
            cmd_by_eg[eg_count] = dict()
            pipe_count, cmd_count = (0, 0)
            for poc in exec_group.get_pipes_and_commands():
                # for each pipe or command (poc)
                # check if it is a pipeline ...
                if isinstance(poc, pipeline_info.PipelineInfo):
                    pipe_count += 1
                    cmd_by_eg[eg_count]['Pipe %s' % pipe_count] = list()
                    for command in poc.get_commands():
                        cmd_by_eg[eg_count]['Pipe %s' % pipe_count].append(
                            command.get_command())
                # ... or a command
                elif isinstance(poc, command_info.CommandInfo):
                    cmd_count += 1
                    cmd_by_eg[eg_count]['Cmd %s' %
                                        cmd_count] = poc.get_command()

        # Set step state back to original state
        self.get_step()._state = previous_state
        return misc.str_to_sha1_b62(json.dumps(cmd_by_eg))[0:8]
示例#5
0
文件: run.py 项目: specht/uap
    def add_temporary_file(self, prefix = '', suffix = '', designation = None):
        '''
        Returns the name of a temporary file (created by tempfile library).
        Name and output directory placeholder are concatenated. The concatenated
        string is returned and stored in a list. The placeholder is immediately
        properly adjusted by @replace_output_dir_du_jour.
        '''
        count = len(self._temp_paths)
        temp_placeholder = str()

        while True:
            hashtag = misc.str_to_sha1_b62('%s.%s.%s' % (prefix, count, suffix))
            temp_name = prefix + hashtag + suffix
            temp_placeholder = os.path.join(
                self.get_output_directory_du_jour_placeholder(), temp_name)

            if not temp_placeholder in self._temp_paths:
                break
            else:
                count += 1
        

        logger.info("Temporary file (#%s): %s" %
              (len(self._temp_paths) + 1, temp_name) )

        # _known_paths dict is logged
        known_paths = dict()
        known_paths[temp_placeholder] = {
            'label': os.path.basename(temp_placeholder),
            'designation': designation,
            'type': ''
        }
        self.add_known_paths(known_paths)
        # _temp_paths list contains all temporary files which are going to be
        # deleted
        self._temp_paths.append(temp_placeholder)
        return temp_placeholder
示例#6
0
文件: run.py 项目: specht/uap
    def get_execution_hashtag(self):
        '''
        Creates a hash tag based on the commands to be executed.

        This causes runs to be marked for rerunning if the commands to be
        executed change.
        '''

        # Store step state
        previous_state = self.get_step()._state
        # Set step state to DECLARING to avoid circular dependencies
        self.get_step()._state = abst.AbstractStep.states.DECLARING

        cmd_by_eg = dict()
        eg_count = 0
        for exec_group in self.get_exec_groups():
            eg_count += 1
            cmd_by_eg[eg_count] = dict()
            pipe_count, cmd_count = (0, 0)
            for poc in exec_group.get_pipes_and_commands():
                # for each pipe or command (poc)
                # check if it is a pipeline ...
                if isinstance(poc, pipeline_info.PipelineInfo):
                    pipe_count += 1
                    cmd_by_eg[eg_count]['Pipe %s' % pipe_count] = list()
                    for command in poc.get_commands():
                        cmd_by_eg[eg_count]['Pipe %s' % pipe_count].append(
                            command.get_command())
                # ... or a command
                elif isinstance(poc, command_info.CommandInfo):
                    cmd_count += 1
                    cmd_by_eg[eg_count]['Cmd %s' % cmd_count] = poc.get_command()

        # Set step state back to original state
        self.get_step()._state = previous_state
        return misc.str_to_sha1_b62(json.dumps(cmd_by_eg))[0:8]