def check_for_preexisting_sample_runs(project_obj, sample_obj, restart_running_jobs, restart_finished_jobs): """If any analysis is undergoing or has completed for this sample's seqruns, raise a RuntimeError. :param NGIProject project_obj: The project object :param NGISample sample_obj: The sample object :param boolean restart_running_jobs: command line parameter :param boolean restart_finished_jobs: command line parameter :raise RuntimeError if the status is RUNNING or DONE and the flags do not allow to continue """ project_id = project_obj.project_id sample_id = sample_obj.name charon_session = CharonSession() sample_libpreps = charon_session.sample_get_libpreps(projectid=project_id, sampleid=sample_id) for libprep in sample_libpreps['libpreps']: libprep_id = libprep['libprepid'] for seqrun in charon_session.libprep_get_seqruns(projectid=project_id, sampleid=sample_id, libprepid=libprep_id)['seqruns']: seqrun_id = seqrun['seqrunid'] aln_status = charon_session.seqrun_get(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id).get('alignment_status') if (aln_status == "RUNNING" and not restart_running_jobs) or \ (aln_status == "DONE" and not restart_finished_jobs): raise RuntimeError('Project/Sample "{}/{}" has a preexisting ' 'seqrun "{}" with status "{}"'.format(project_obj, sample_obj, seqrun_id, aln_status))
def get_finished_seqruns_for_sample(project_id, sample_id, include_failed_libpreps=False): """Find all the finished seqruns for a particular sample. :param str project_id: The id of the project :param str sample_id: The id of the sample :returns: A dict of {libprep_01: [seqrun_01, ..., seqrun_nn], ...} :rtype: dict """ charon_session = CharonSession() sample_libpreps = charon_session.sample_get_libpreps(projectid=project_id, sampleid=sample_id) libpreps = collections.defaultdict(list) for libprep in sample_libpreps['libpreps']: if libprep.get('qc') != "FAILED" or include_failed_libpreps: libprep_id = libprep['libprepid'] for seqrun in charon_session.libprep_get_seqruns(projectid=project_id, sampleid=sample_id, libprepid=libprep_id)['seqruns']: seqrun_id = seqrun['seqrunid'] aln_status = charon_session.seqrun_get(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id).get('alignment_status') if aln_status == "DONE": libpreps[libprep_id].append(seqrun_id) else: LOG.debug('Skipping seqrun "{}" due to alignment_status ' '"{}"'.format(seqrun_id, aln_status)) else: LOG.info('Skipping libprep "{}" due to qc status ' '"{}"'.format(libprep, libprep.get("qc"))) return dict(libpreps)
def get_valid_seqruns_for_sample(project_id, sample_id, include_failed_libpreps=False, include_done_seqruns=False, status_field="alignment_status"): """Find all the valid seqruns for a particular sample. :param str project_id: The id of the project :param str sample_id: The id of the sample :param bool include_failed_libpreps: Include seqruns for libreps that have failed QC :param bool include_done_seqruns: Include seqruns that are already marked DONE :returns: A dict of {libprep_01: [seqrun_01, ..., seqrun_nn], ...} :rtype: dict :raises ValueError: If status_field is not a valid value """ valid_status_values = ( "alignment_status", "genotype_status", ) if status_field not in valid_status_values: raise ValueError('"status_field" argument must be one of {} ' '(value passed was "{}")'.format( ", ".join(valid_status_values), status_field)) charon_session = CharonSession() sample_libpreps = charon_session.sample_get_libpreps(projectid=project_id, sampleid=sample_id) libpreps = collections.defaultdict(list) for libprep in sample_libpreps['libpreps']: if libprep.get('qc') != "FAILED" or include_failed_libpreps: libprep_id = libprep['libprepid'] for seqrun in charon_session.libprep_get_seqruns( projectid=project_id, sampleid=sample_id, libprepid=libprep_id)['seqruns']: seqrun_id = seqrun['seqrunid'] try: aln_status = charon_session.seqrun_get( projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id)[status_field] except KeyError: LOG.error( 'Field "{}" not available for seqrun "{}" in Charon ' 'for project "{}" / sample "{}". Including as ' 'valid.'.format(status_field, seqrun_id, project_id, sample_id)) aln_status = None if aln_status != "DONE" or include_done_seqruns: libpreps[libprep_id].append(seqrun_id) else: LOG.info('Skipping seqrun "{}" due to {}' '"{}"'.format(seqrun_id, status_field, aln_status)) else: LOG.info('Skipping libprep "{}" due to qc status ' '"{}"'.format(libprep, libprep.get("qc"))) return dict(libpreps)
def write_to_charon_alignment_results(base_path, project_name, project_id, sample_id, libprep_id, seqrun_id): """Update the status of a sequencing run after alignment. :param str project_name: The name of the project (e.g. T.Durden_14_01) :param str project_id: The id of the project (e.g. P1171) :param str sample_id: ... :param str libprep_id: ... :param str seqrun_id: ... :raises RuntimeError: If the Charon database could not be updated :raises ValueError: If the output data could not be parsed. """ charon_session = CharonSession() try: seqrun_dict = charon_session.seqrun_get(project_id, sample_id, libprep_id, seqrun_id) except CharonError as e: raise CharonError('Error accessing database for project "{}", sample {}; ' 'could not update Charon while performing best practice: ' '{}'.format(project_name, sample_id, e)) piper_run_id = seqrun_id.split("_")[3] seqrun_dict["lanes"] = 0 if seqrun_dict.get("alignment_status") == "DONE": LOG.warn("Sequencing run \"{}\" marked as DONE but writing new alignment results; " "this will overwrite the previous results.".format(seqrun_id)) # Find all the appropriate files piper_result_dir = os.path.join(base_path, "ANALYSIS", project_name, "02_preliminary_alignment_qc") try: os.path.isdir(piper_result_dir) and os.listdir(piper_result_dir) except OSError as e: raise ValueError("Piper result directory \"{}\" inaccessible when updating stats to Charon: {}.".format(piper_result_dir, e)) piper_qc_dir_base = "{}.{}.{}".format(sample_id, piper_run_id, sample_id) piper_qc_path = "{}*/".format(os.path.join(piper_result_dir, piper_qc_dir_base)) piper_qc_dirs = glob.glob(piper_qc_path) if not piper_qc_dirs: # Something went wrong in the alignment or we can't parse the file format raise ValueError("Piper qc directories under \"{}\" are missing or in an unexpected format when updating stats to Charon.".format(piper_qc_path)) # Examine each lane and update the dict with its alignment metrics for qc_lane in piper_qc_dirs: genome_result = os.path.join(qc_lane, "genome_results.txt") # This means that if any of the lanes are missing results, the sequencing run is marked as a failure. # We should flag this somehow and send an email at some point. if not os.path.isfile(genome_result): raise ValueError("File \"genome_results.txt\" is missing from Piper result directory \"{}\"".format(piper_result_dir)) # Get the alignment results for this lane lane_alignment_metrics = parse_qualimap_results(genome_result) # Update the dict for this lane update_seq_run_for_lane(seqrun_dict, lane_alignment_metrics) try: # Update the seqrun in the Charon database charon_session.seqrun_update(**seqrun_dict) except CharonError as e: error_msg = ('Failed to update run alignment status for run "{}" in project {} ' 'sample {}, library prep {} to Charon database: {}'.format(seqrun_id, project_name, sample_id, libprep_id, e)) raise CharonError(error_msg)
def analyze(analysis_object, config=None, config_file_path=None): charon_session = CharonSession() charon_pj=charon_session.project_get(analysis_object.project.project_id) reference_genome=charon_pj.get('reference') if charon_pj.get("sequencing_facility") == "NGI-S": analysis_object.sequencing_facility="sthlm" elif charon_pj.get("sequencing_facility") == "NGI-U": analysis_object.sequencing_facility="upps" else: LOG.error("charon project not registered with stockholm or uppsala. Which config file should we use for the RNA pipeline ?") raise RuntimeError fastq_files=[] if reference_genome and reference_genome != 'other': for sample in analysis_object.project: try: charon_reported_status = charon_session.sample_get(analysis_object.project.project_id, sample).get('analysis_status') # Check Charon to ensure this hasn't already been processed do_analyze=handle_sample_status(analysis_object, sample, charon_reported_status) if not do_analyze : continue except CharonError as e: LOG.error(e) for libprep in sample: charon_lp_status=charon_session.libprep_get(analysis_object.project.project_id, sample.name, libprep.name).get('qc') do_analyze=handle_libprep_status(analysis_object, libprep, charon_lp_status) if not do_analyze : continue else: for seqrun in libprep: charon_sr_status=charon_session.seqrun_get(analysis_object.project.project_id, sample.name, libprep.name, seqrun.name).get('alignment_status') do_analyze=handle_seqrun_status(analysis_object, seqrun, charon_sr_status) if not do_analyze : continue else: seqrun.being_analyzed=True sample.being_analyzed = sample.being_analyzed or True # filter out index files from analysis for fastq_file in filter(lambda f: not is_index_file(f), seqrun.fastq_files): fastq_path=os.path.join(analysis_object.project.base_path, "DATA", analysis_object.project.project_id, sample.name, libprep.name, seqrun.name, fastq_file) fastq_files.append(fastq_path) if not fastq_files: LOG.error("No fastq files obtained for the analysis fo project {}, please check the Charon status.".format(analysis_object.project.name)) else : if analysis_object.restart_running_jobs: stop_ongoing_analysis(analysis_object) fastq_dir=preprocess_analysis(analysis_object, fastq_files) sbatch_path=write_batch_job(analysis_object, reference_genome, fastq_dir) job_id=start_analysis(sbatch_path) analysis_path=os.path.join(analysis_object.project.base_path, "ANALYSIS", analysis_object.project.project_id, 'rna_ngi') record_project_job(analysis_object.project, job_id, analysis_path)
def get_valid_seqruns_for_sample(project_id, sample_id, include_failed_libpreps=False, include_done_seqruns=False, status_field="alignment_status"): """Find all the valid seqruns for a particular sample. :param str project_id: The id of the project :param str sample_id: The id of the sample :param bool include_failed_libpreps: Include seqruns for libreps that have failed QC :param bool include_done_seqruns: Include seqruns that are already marked DONE :returns: A dict of {libprep_01: [seqrun_01, ..., seqrun_nn], ...} :rtype: dict :raises ValueError: If status_field is not a valid value """ valid_status_values = ("alignment_status", "genotype_status",) if status_field not in valid_status_values: raise ValueError('"status_field" argument must be one of {} ' '(value passed was "{}")'.format(", ".join(valid_status_values), status_field)) charon_session = CharonSession() sample_libpreps = charon_session.sample_get_libpreps(projectid=project_id, sampleid=sample_id) libpreps = collections.defaultdict(list) for libprep in sample_libpreps['libpreps']: if libprep.get('qc') != "FAILED" or include_failed_libpreps: libprep_id = libprep['libprepid'] for seqrun in charon_session.libprep_get_seqruns(projectid=project_id, sampleid=sample_id, libprepid=libprep_id)['seqruns']: seqrun_id = seqrun['seqrunid'] try: aln_status = charon_session.seqrun_get(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id)[status_field] except KeyError: LOG.error('Field "{}" not available for seqrun "{}" in Charon ' 'for project "{}" / sample "{}". Including as ' 'valid.'.format(status_field, seqrun_id, project_id, sample_id)) aln_status = None if aln_status != "DONE" or include_done_seqruns: libpreps[libprep_id].append(seqrun_id) else: LOG.info('Skipping seqrun "{}" due to {}' '"{}"'.format(seqrun_id,status_field, aln_status)) else: LOG.info('Skipping libprep "{}" due to qc status ' '"{}"'.format(libprep, libprep.get("qc"))) return dict(libpreps)
def analyze(analysis_object, config=None, config_file_path=None): charon_session = CharonSession() charon_pj = charon_session.project_get(analysis_object.project.project_id) reference_genome = charon_pj.get('reference') if charon_pj.get("sequencing_facility") == "NGI-S": analysis_object.sequencing_facility = "sthlm" elif charon_pj.get("sequencing_facility") == "NGI-U": analysis_object.sequencing_facility = "upps" else: LOG.error( "charon project not registered with stockholm or uppsala. Which config file should we use for the RNA pipeline ?" ) raise RuntimeError fastq_files = [] if reference_genome and reference_genome != 'other': for sample in analysis_object.project: try: charon_reported_status = charon_session.sample_get( analysis_object.project.project_id, sample).get('analysis_status') # Check Charon to ensure this hasn't already been processed do_analyze = handle_sample_status(analysis_object, sample, charon_reported_status) if not do_analyze: continue except CharonError as e: LOG.error(e) for libprep in sample: charon_lp_status = charon_session.libprep_get( analysis_object.project.project_id, sample.name, libprep.name).get('qc') do_analyze = handle_libprep_status(analysis_object, libprep, charon_lp_status) if not do_analyze: continue else: for seqrun in libprep: charon_sr_status = charon_session.seqrun_get( analysis_object.project.project_id, sample.name, libprep.name, seqrun.name).get('alignment_status') do_analyze = handle_seqrun_status( analysis_object, seqrun, charon_sr_status) if not do_analyze: continue else: seqrun.being_analyzed = True sample.being_analyzed = sample.being_analyzed or True for fastq_file in seqrun.fastq_files: fastq_path = os.path.join( analysis_object.project.base_path, "DATA", analysis_object.project.project_id, sample.name, libprep.name, seqrun.name, fastq_file) fastq_files.append(fastq_path) if not fastq_files: LOG.error( "No fastq files obtained for the analysis fo project {}, please check the Charon status." .format(analysis_object.project.name)) else: if analysis_object.restart_running_jobs: stop_ongoing_analysis(analysis_object) fastq_dir = preprocess_analysis(analysis_object, fastq_files) sbatch_path = write_batch_job(analysis_object, reference_genome, fastq_dir) job_id = start_analysis(sbatch_path) analysis_path = os.path.join(analysis_object.project.base_path, "ANALYSIS", analysis_object.project.project_id, 'rna_ngi') record_project_job(analysis_object.project, job_id, analysis_path)
def update_charon_with_local_jobs_status(): """Check the status of all locally-tracked jobs and update Charon accordingly. """ LOG.info("Updating Charon with the status of all locally-tracked jobs...") with get_db_session() as session: charon_session = CharonSession() # Sequencing Run Analyses for seqrun_entry in session.query(SeqrunAnalysis).all(): # Local names workflow = seqrun_entry.workflow project_name = seqrun_entry.project_name project_id = seqrun_entry.project_id project_base_path = seqrun_entry.project_base_path sample_id = seqrun_entry.sample_id libprep_id = seqrun_entry.libprep_id seqrun_id = seqrun_entry.seqrun_id pid = seqrun_entry.process_id exit_code = get_exit_code(workflow_name=workflow, project_base_path=project_base_path, project_name=project_name, sample_id=sample_id, libprep_id=libprep_id, seqrun_id=seqrun_id) label = "project/sample/libprep/seqrun {}/{}/{}/{}".format(project_name, sample_id, libprep_id, seqrun_id) try: if exit_code == 0: # 0 -> Job finished successfully LOG.info('Workflow "{}" for {} finished succesfully. ' 'Recording status "DONE" in Charon'.format(workflow, label)) set_alignment_status = "DONE" try: write_to_charon_alignment_results(base_path=project_base_path, project_name=project_name, project_id=project_id, sample_id=sample_id, libprep_id=libprep_id, seqrun_id=seqrun_id) except (RuntimeError, ValueError) as e: LOG.error(e) set_alignment_status = "FAILED" charon_session.seqrun_update(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id, alignment_status=set_alignment_status) # Job is only deleted if the Charon update succeeds session.delete(seqrun_entry) elif exit_code == 1 or (not psutil.pid_exists(pid) and not exit_code): if exit_code == 1: # 1 -> Job failed (DATA_FAILURE / COMPUTATION_FAILURE ?) LOG.info('Workflow "{}" for {} failed. Recording status ' '"FAILED" in Charon.'.format(workflow, label)) else: # Job failed without writing an exit code (process no longer running) LOG.error('ERROR: No exit code found for process {} ' 'but it does not appear to be running ' '(pid {} does not exist). Setting status to ' '"FAILED", inspect manually'.format(label, pid)) charon_session.seqrun_update(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id, alignment_status="FAILED") # Job is only deleted if the Charon update succeeds LOG.debug("Deleting local entry {}".format(seqrun_entry)) session.delete(seqrun_entry) else: # None -> Job still running charon_status = charon_session.seqrun_get(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id)['alignment_status'] if not charon_status == "RUNNING": LOG.warn('Tracking inconsistency for {}: Charon status is "{}" but ' 'local process tracking database indicates it is running. ' 'Setting value in Charon to RUNNING.'.format(label, charon_status)) charon_session.seqrun_update(projectid=project_id, sampleid=sample_id, libprepid=libprep_id, seqrunid=seqrun_id, alignment_status="RUNNING") except CharonError as e: LOG.error('Unable to update Charon status for "{}": {}'.format(label, e)) for sample_entry in session.query(SampleAnalysis).all(): # Local names workflow = sample_entry.workflow project_name = sample_entry.project_name project_id = sample_entry.project_id project_base_path = sample_entry.project_base_path sample_id = sample_entry.sample_id pid = sample_entry.process_id exit_code = get_exit_code(workflow_name=workflow, project_base_path=project_base_path, project_name=project_name, sample_id=sample_id) label = "project/sample/libprep/seqrun {}/{}".format(project_name, sample_id) try: if exit_code == 0: # 0 -> Job finished successfully LOG.info('Workflow "{}" for {} finished succesfully. ' 'Recording status "DONE" in Charon'.format(workflow, label)) set_status = "DONE" ## TODO implement sample-level analysis results parsing / reporting to Charon? #try: # write_to_charon_alignment_results(base_path=project_base_path, # project_name=project_name, # project_id=project_id, # sample_id=sample_id, # libprep_id=libprep_id, # seqrun_id=seqrun_id) #except (RuntimeError, ValueError) as e: # LOG.error(e) # set_alignment_status = "FAILED" charon_session.sample_update(projectid=project_id, sampleid=sample_id, status=set_status) # Job is only deleted if the Charon update succeeds session.delete(sample_entry) elif exit_code == 1 or (not psutil.pid_exists(pid) and not exit_code): if exit_code == 1: # 1 -> Job failed (DATA_FAILURE / COMPUTATION_FAILURE ?) LOG.info('Workflow "{}" for {} failed. Recording status ' '"COMPUTATION_FAILED" in Charon.'.format(workflow, label)) else: # Job failed without writing an exit code LOG.error('ERROR: No exit code found for process {} ' 'but it does not appear to be running ' '(pid {} does not exist). Setting status to ' '"COMPUTATION_FAILED", inspect manually'.format(label, pid)) charon_session.sample_update(projectid=project_id, sampleid=sample_id, status="COMPUTATION_FAILED") # Job is only deleted if the Charon update succeeds session.delete(sample_entry) else: # None -> Job still running try: charon_status = charon_session.sample_get(projectid=project_id, sampleid=sample_id)['status'] except (CharonError, KeyError) as e: LOG.warn('Unable to get required information from Charon for ' 'sample "{}" / project "{}" -- forcing it to RUNNING: {}'.format(sample_id, project_id, e)) charon_status = "NEW" if not charon_status == "RUNNING": LOG.warn('Tracking inconsistency for {}: Charon status is "{}" but ' 'local process tracking database indicates it is running. ' 'Setting value in Charon to RUNNING.'.format(label, charon_status)) charon_session.sample_update(projectid=project_id, sampleid=sample_id, status="RUNNING") except CharonError as e: LOG.error('Unable to update Charon status for "{}": {}'.format(label, e)) session.commit()
def launch_analysis(level, projects_to_analyze, restart_failed_jobs=False, config=None, config_file_path=None): """Launch the appropriate seqrun (flowcell-level) analysis for each fastq file in the project. :param list projects_to_analyze: The list of projects (Project objects) to analyze :param dict config: The parsed NGI configuration file; optional/has default. :param str config_file_path: The path to the NGI configuration file; optional/has default. """ # Update Charon with the local state of all the jobs we're running update_charon_with_local_jobs_status() charon_session = CharonSession() for project in projects_to_analyze: # Get information from Charon regarding which workflows to run try: # E.g. "NGI" for NGI DNA Samples workflow = charon_session.project_get(project.project_id)["pipeline"] except (KeyError, CharonError) as e: # Workflow missing from Charon? LOG.error('Skipping project "{}" because of error: {}'.format(project, e)) continue try: analysis_engine_module_name = config["analysis"]["workflows"][workflow]["analysis_engine"] except KeyError: error_msg = ("No analysis engine for workflow \"{}\" specified " "in configuration file. Skipping this workflow " "for project {}".format(workflow, project)) LOG.error(error_msg) raise RuntimeError(error_msg) # Import the adapter module specified in the config file (e.g. piper_ngi) try: analysis_module = importlib.import_module(analysis_engine_module_name) except ImportError as e: error_msg = ('Skipping project "{}" workflow "{}": couldn\'t import ' 'module "{}": {}'.format(project, workflow, analysis_engine_module_name, e)) LOG.error(error_msg) # Next project continue # This is weird objects_to_process = [] if level == "sample": for sample in project: objects_to_process.append({"project": project, "sample": sample}) elif level == "seqrun": for sample in project: for libprep in sample: for seqrun in libprep: objects_to_process.append({"project": project, "sample": sample, "libprep": libprep, "seqrun": seqrun}) # Still weird and not so great for obj_dict in objects_to_process: project = obj_dict.get("project") sample = obj_dict.get("sample") libprep = obj_dict.get("libprep") seqrun = obj_dict.get("seqrun") try: if level == "seqrun": charon_reported_status = charon_session.seqrun_get(project.project_id, sample, libprep, seqrun)['alignment_status'] else: # sample-level charon_reported_status = charon_session.sample_get(project.project_id, sample)['status'] except (CharonError, KeyError) as e: LOG.warn('Unable to get required information from Charon for ' 'sample "{}" / project "{}" -- forcing it to new: {}'.format(sample, project, e)) if level == "seqrun": charon_session.seqrun_update(project.project_id, sample.name, libprep.name, seqrun.name, alignment_status="NEW") charon_reported_status = charon_session.seqrun_get(project.project_id, sample, libprep, seqrun)['alignment_status'] else: charon_session.sample_update(project.project_id, sample.name, status="NEW") charon_reported_status = charon_session.sample_get(project.project_id, sample)['status'] # Check Charon to ensure this hasn't already been processed if charon_reported_status in ("RUNNING", "DONE"): if level == "seqrun": LOG.info('Charon reports seqrun analysis for project "{}" / sample "{}" ' '/ libprep "{}" / seqrun "{}" does not need processing ' ' (already "{}")'.format(project, sample, libprep, seqrun, charon_reported_status)) else: # Sample LOG.info('Charon reports seqrun analysis for project "{}" / sample "{}" ' 'does not need processing ' ' (already "{}")'.format(project, sample, charon_reported_status)) continue elif charon_reported_status == "FAILED": if not restart_failed_jobs: if level == "seqrun": LOG.error('FAILED: Project "{}" / sample "{}" / library "{}" ' '/ flowcell "{}": Charon reports FAILURE, manual ' 'investigation needed!'.format(project, sample, libprep, seqrun)) else: # Sample LOG.error('FAILED: Project "{}" / sample "{}" Charon reports FAILURE, manual ' 'investigation needed!'.format(project, sample, libprep, seqrun)) continue try: # The engines themselves know which sub-workflows # they need to execute for a given level. For example, # with DNA Variant Calling on the sequencing run # level, we need to execute basic alignment and QC. if level == "seqrun": LOG.info('Attempting to launch seqrun analysis for ' 'project "{}" / sample "{}" / libprep "{}" ' '/ seqrun "{}", workflow "{}"'.format(project, sample, libprep, seqrun, workflow)) analysis_module.analyze_seqrun(project=project, sample=sample, libprep=libprep, seqrun=seqrun) else: # sample level LOG.info('Attempting to launch sample analysis for ' 'project "{}" / sample "{}" / workflow ' '"{}"'.format(project, sample, workflow)) analysis_module.analyze_sample(project=project, sample=sample) except Exception as e: raise LOG.error('Cannot process project "{}" / sample "{}" / ' 'libprep "{}" / seqrun "{}" / workflow ' '"{}" : {}'.format(project, sample, libprep, seqrun, workflow, e)) set_new_seqrun_status = "FAILED" continue