def test_get_slurm_job_status(self, mock_subprocess):
mock_subprocess.side_effect = [
'PENDING', 'COMPLETED', 'CANCELLED', '', 'Whut'
]
got_job_status = slurm.get_slurm_job_status(self.slurm_job_id)
self.assertIsNone(got_job_status)
got_job_status = slurm.get_slurm_job_status(self.slurm_job_id)
self.assertEqual(got_job_status, 0)
got_job_status = slurm.get_slurm_job_status(self.slurm_job_id)
self.assertEqual(got_job_status, 1)
with self.assertRaises(ValueError):
got_job_status = slurm.get_slurm_job_status(self.slurm_job_id)
with self.assertRaises(RuntimeError):
got_job_status = slurm.get_slurm_job_status(self.slurm_job_id)
def update_charon_with_local_jobs_status(quiet=False, config=None, config_file_path=None):
"""Check the status of all locally-tracked jobs and update Charon accordingly.
"""
if quiet and not config.get("quiet"):
config['quiet'] = True
LOG.info("Updating Charon with the status of all locally-tracked jobs...")
with get_db_session() as session:
charon_session = CharonSession()
for sample_entry in session.query(SampleAnalysis).all():
# Local names
workflow = sample_entry.workflow
project_name = sample_entry.project_name
project_id = sample_entry.project_id
project_base_path = sample_entry.project_base_path
sample_id = sample_entry.sample_id
engine = sample_entry.engine
# Only one of these id fields (slurm, pid) will have a value
slurm_job_id = sample_entry.slurm_job_id
process_id = sample_entry.process_id
piper_exit_code = get_exit_code(workflow_name=workflow,
project_base_path=project_base_path,
project_name=project_name,
project_id=project_id,
sample_id=sample_id)
label = "project/sample {}/{}".format(project_name, sample_id)
if workflow not in ("merge_process_variantcall", "genotype_concordance",):
LOG.error('Unknown workflow "{}" for {}; cannot update '
'Charon. Skipping sample.'.format(workflow, label))
continue
try:
project_obj = create_project_obj_from_analysis_log(project_name,
project_id,
project_base_path,
sample_id,
workflow)
except IOError as e: # analysis log file is missing!
error_text = ('Could not find analysis log file! Cannot update '
'Charon for {} run {}/{}: {}'.format(workflow,
project_id,
sample_id,
e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="ERROR",
info_text=error_text,
workflow=workflow)
continue
try:
if piper_exit_code == 0:
# 0 -> Job finished successfully
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
set_status = "ANALYZED" # sample level
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
set_status = "DONE" # sample level
recurse_status = "DONE" # For the seqrun level
info_text = ('Workflow "{}" for {} finished succesfully. '
'Recording status {} in Charon'.format(workflow,
label,
set_status))
LOG.info(info_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="INFO",
info_text=info_text,
workflow=workflow)
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=recurse_status,
config=config)
# Job is only deleted if the Charon status update succeeds
session.delete(sample_entry)
#run MultiQC
LOG.info("Running MultiQC on project {}".format(project_name))
try:
run_multiqc(project_base_path, project_id, project_name)
except Exception as e:
LOG.error(e)
if workflow == "merge_process_variantcall":
# Parse seqrun output results / update Charon
# This is a semi-optional step -- failure here will send an
# email but not more than once. The record is still removed
# from the local jobs database, so this will have to be done
# manually if you want it done at all.
piper_qc_dir = os.path.join(project_base_path, "ANALYSIS",
project_id, "piper_ngi",
"02_preliminary_alignment_qc")
update_coverage_for_sample_seqruns(project_id, sample_id,
piper_qc_dir)
update_sample_duplication_and_coverage(project_id, sample_id,
project_base_path)
elif workflow == "genotype_concordance":
piper_gt_dir = os.path.join(project_base_path, "ANALYSIS",
project_id, "piper_ngi",
"03_genotype_concordance")
try:
update_gtc_for_sample(project_id, sample_id, piper_gt_dir)
except (CharonError, IOError, ValueError) as e:
LOG.error(e)
elif type(piper_exit_code) is int and piper_exit_code > 0:
# 1 -> Job failed
set_status = "FAILED"
error_text = ('Workflow "{}" for {} failed. Recording status '
'{} in Charon.'.format(workflow, label, set_status))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="ERROR",
info_text=error_text,
workflow=workflow)
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj, status_field=seqrun_status_field,
status_value=set_status, config=config)
# Job is only deleted if the Charon update succeeds
session.delete(sample_entry)
else:
# None -> Job still running OR exit code was never written (failure)
JOB_FAILED = None
if slurm_job_id:
try:
slurm_exit_code = get_slurm_job_status(slurm_job_id)
except ValueError as e:
slurm_exit_code = 1
if slurm_exit_code is not None: # "None" indicates job is still running
JOB_FAILED = True
else:
if not psutil.pid_exists(process_id):
# Job did not write an exit code and is also not running
JOB_FAILED = True
if JOB_FAILED:
set_status = "FAILED"
error_text = ('No exit code found but job not running '
'for {} / {}: setting status to {} in '
'Charon'.format(label, workflow, set_status))
if slurm_job_id:
exit_code_file_path = \
create_exit_code_file_path(workflow_subtask=workflow,
project_base_path=project_base_path,
project_name=project_name,
project_id=project_id,
sample_id=sample_id)
error_text += (' (slurm job id "{}", exit code file path '
'"{}")'.format(slurm_job_id, exit_code_file_path))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine, level="ERROR",
info_text=error_text,
workflow=workflow)
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=set_status,
config=config)
# Job is only deleted if the Charon update succeeds
LOG.debug("Deleting local entry {}".format(sample_entry))
session.delete(sample_entry)
else: # Job still running
set_status = "UNDER_ANALYSIS"
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
recurse_status = "RUNNING"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
recurse_status = "UNDER_ANALYSIS"
try:
charon_status = \
charon_session.sample_get(projectid=project_id,
sampleid=sample_id).get(sample_status_field)
if charon_status and not charon_status == set_status:
LOG.warn('Tracking inconsistency for {}: Charon status '
'for field "{}" is "{}" but local process tracking '
'database indicates it is running. Setting value '
'in Charon to {}.'.format(label, sample_status_field,
charon_status, set_status))
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=recurse_status,
config=config)
except CharonError as e:
error_text = ('Unable to update/verify Charon '
'for {}: {}'.format(label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
except CharonError as e:
error_text = ('Unable to update Charon for {}: '
'{}'.format(label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
except OSError as e:
error_text = ('Permissions error when trying to update Charon '
'"{}" status for "{}": {}'.format(workflow, label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
session.commit()
def analyze(analysis_object,
level='sample',
config=None,
config_file_path=None):
"""Analyze data at the sample level.
:param NGIAnalysis analysis_object: holds all the parameters for the analysis
:raises ValueError: If exec_mode is an unsupported value
"""
charon_session = CharonSession()
for sample in analysis_object.project:
try:
charon_reported_status = charon_session.sample_get(
analysis_object.project.project_id,
sample).get('analysis_status')
# Check Charon to ensure this hasn't already been processed
do_analyze = handle_sample_status(analysis_object, sample,
charon_reported_status)
if not do_analyze:
continue
except CharonError as e:
LOG.error(e)
continue
if level == "sample":
status_field = "alignment_status"
elif level == "genotype":
status_field = "genotype_status"
else:
LOG.warn('Unknown workflow level: "{}"'.format(level))
status_field = "alignment_status" # Or should we abort?
try:
check_for_preexisting_sample_runs(
analysis_object.project, sample,
analysis_object.restart_running_jobs,
analysis_object.restart_finished_jobs, status_field)
except RuntimeError as e:
raise RuntimeError(
'Aborting processing of project/sample "{}/{}": '
'{}'.format(analysis_object.project, sample, e))
if analysis_object.exec_mode.lower() not in ("sbatch", "local"):
raise ValueError(
'"exec_mode" param must be one of "sbatch" or "local" '
'value was "{}"'.format(analysis_object.exec_mode))
if analysis_object.exec_mode == "local":
modules_to_load = analysis_object.config.get("piper", {}).get(
"load_modules", [])
load_modules(modules_to_load)
for workflow_subtask in workflows.get_subtasks_for_level(level=level):
if level == "genotype":
genotype_status = None # Some records in Charon lack this field, I'm guessing
try:
charon_session = CharonSession()
genotype_status = charon_session.sample_get(
projectid=analysis_object.project.project_id,
sampleid=sample.name).get("genotype_status")
except CharonError as e:
LOG.error(
'Couldn\'t determine genotyping status for project/'
'sample "{}/{}"; skipping analysis.'.format(
analysis_object.project, sample))
continue
if find_previous_genotype_analyses(
analysis_object.project,
sample) or genotype_status == "DONE":
if not analysis_object.restart_finished_jobs:
LOG.info(
'Project/sample "{}/{}" has completed genotype '
'analysis previously; skipping (use flag to force '
'analysis)'.format(analysis_object.project,
sample))
continue
if analysis_object.restart_running_jobs:
# Kill currently-running jobs if they exist
kill_running_sample_analysis(
workflow_subtask=workflow_subtask,
project_id=analysis_object.project.project_id,
sample_id=sample.name)
# This checks the local jobs database
if not is_sample_analysis_running_local(
workflow_subtask=workflow_subtask,
project_id=analysis_object.project.project_id,
sample_id=sample.name):
LOG.info('Launching "{}" analysis for sample "{}" in project '
'"{}"'.format(workflow_subtask, sample,
analysis_object.project))
try:
log_file_path = create_log_file_path(
workflow_subtask=workflow_subtask,
project_base_path=analysis_object.project.base_path,
project_name=analysis_object.project.dirname,
project_id=analysis_object.project.project_id,
sample_id=sample.name)
rotate_file(log_file_path)
exit_code_path = create_exit_code_file_path(
workflow_subtask=workflow_subtask,
project_base_path=analysis_object.project.base_path,
project_name=analysis_object.project.dirname,
project_id=analysis_object.project.project_id,
sample_id=sample.name)
if level == "sample":
if not analysis_object.keep_existing_data:
remove_previous_sample_analyses(
analysis_object.project, sample)
default_files_to_copy = None
elif level == "genotype":
if not analysis_object.keep_existing_data:
remove_previous_genotype_analyses(
analysis_object.project)
default_files_to_copy = None
# Update the project to keep only valid fastq files for setup.xml creation
if level == "genotype":
updated_project, default_files_to_copy = \
collect_files_for_sample_analysis(analysis_object.project,
sample,
restart_finished_jobs=True,
status_field="genotype_status")
else:
updated_project, default_files_to_copy = \
collect_files_for_sample_analysis(analysis_object.project,
sample,
analysis_object.restart_finished_jobs,
status_field="alignment_status")
setup_xml_cl, setup_xml_path = build_setup_xml(
project=updated_project,
sample=sample,
workflow=workflow_subtask,
local_scratch_mode=(
analysis_object.exec_mode == "sbatch"),
config=analysis_object.config)
piper_cl = build_piper_cl(
project=analysis_object.project,
workflow_name=workflow_subtask,
setup_xml_path=setup_xml_path,
exit_code_path=exit_code_path,
config=analysis_object.config,
exec_mode=analysis_object.exec_mode,
generate_bqsr_bam=analysis_object.generate_bqsr_bam)
if analysis_object.exec_mode == "sbatch":
process_id = None
slurm_job_id = sbatch_piper_sample(
[setup_xml_cl, piper_cl],
workflow_subtask,
analysis_object.project,
sample,
restart_finished_jobs=analysis_object.
restart_finished_jobs,
files_to_copy=default_files_to_copy)
for x in xrange(10):
# Time delay to let sbatch get its act together
# (takes a few seconds to be visible with sacct)
try:
get_slurm_job_status(slurm_job_id)
break
except ValueError:
time.sleep(2)
else:
LOG.error('sbatch file for sample {}/{} did not '
'queue properly! Job ID {} cannot be '
'found.'.format(analysis_object.project,
sample, slurm_job_id))
else: # "local"
raise NotImplementedError(
'Local execution not currently implemented. '
'I\'m sure Denis can help you with this.')
#slurm_job_id = None
#launch_piper_job(setup_xml_cl, project)
#process_handle = launch_piper_job(piper_cl, project)
#process_id = process_handle.pid
try:
record_process_sample(
project=analysis_object.project,
sample=sample,
analysis_module_name="piper_ngi",
slurm_job_id=slurm_job_id,
process_id=process_id,
workflow_subtask=workflow_subtask)
except RuntimeError as e:
LOG.error(e)
## Question: should we just kill the run in this case or let it go?
continue
except (NotImplementedError, RuntimeError, ValueError) as e:
error_msg = (
'Processing project "{}" / sample "{}" / workflow "{}" '
'failed: {}'.format(analysis_object.project, sample,
workflow_subtask, e))
LOG.error(error_msg)
def analyze(project, sample,
exec_mode="sbatch",
restart_finished_jobs=False,
restart_running_jobs=False,
keep_existing_data=False,
level="sample",
genotype_file=None,
config=None, config_file_path=None,
generate_bqsr_bam=False):
"""Analyze data at the sample level.
:param NGIProject project: the project to analyze
:param NGISample sample: the sample to analyzed
:param str exec_mode: "sbatch" or "local" (local not implemented)
:param bool restart_finished_jobs: Restart jobs that are already done (have a .done file)
:param bool restart_running_jobs: Kill and restart currently-running jobs
:param str level: The level on which to perform the analysis ("sample" or "genotype")
:param str genotype_file: The path to the genotype file (only relevant for genotype analysis)
:param dict config: The parsed configuration file (optional)
:param str config_file_path: The path to the configuration file (optional)
:raises ValueError: If exec_mode is an unsupported value
"""
if level == "sample":
status_field = "alignment_status"
elif level == "genotype":
status_field = "genotype_status"
else:
LOG.warn('Unknown workflow level: "{}"'.format(level))
status_field = "alignment_status" # Or should we abort?
try:
check_for_preexisting_sample_runs(project, sample, restart_running_jobs,
restart_finished_jobs, status_field)
except RuntimeError as e:
raise RuntimeError('Aborting processing of project/sample "{}/{}": '
'{}'.format(project, sample, e))
if exec_mode.lower() not in ("sbatch", "local"):
raise ValueError('"exec_mode" param must be one of "sbatch" or "local" '
'value was "{}"'.format(exec_mode))
if exec_mode == "local":
modules_to_load = config.get("piper", {}).get("load_modules", [])
load_modules(modules_to_load)
for workflow_subtask in workflows.get_subtasks_for_level(level=level):
if level == "genotype":
genotype_status = None # Some records in Charon lack this field, I'm guessing
try:
charon_session = CharonSession()
genotype_status = charon_session.sample_get(projectid=project.project_id,
sampleid=sample.name).get("genotype_status")
except CharonError as e:
LOG.error('Couldn\'t determine genotyping status for project/'
'sample "{}/{}"; skipping analysis.'.format(project, sample))
continue
if find_previous_genotype_analyses(project, sample) or genotype_status == "DONE":
if not restart_finished_jobs:
LOG.info('Project/sample "{}/{}" has completed genotype '
'analysis previously; skipping (use flag to force '
'analysis)'.format(project, sample))
continue
if restart_running_jobs:
# Kill currently-running jobs if they exist
kill_running_sample_analysis(workflow_subtask=workflow_subtask,
project_id=project.project_id,
sample_id=sample.name)
# This checks the local jobs database
if not is_sample_analysis_running_local(workflow_subtask=workflow_subtask,
project_id=project.project_id,
sample_id=sample.name):
LOG.info('Launching "{}" analysis for sample "{}" in project '
'"{}"'.format(workflow_subtask, sample, project))
try:
log_file_path = create_log_file_path(workflow_subtask=workflow_subtask,
project_base_path=project.base_path,
project_name=project.dirname,
project_id=project.project_id,
sample_id=sample.name)
rotate_file(log_file_path)
exit_code_path = create_exit_code_file_path(workflow_subtask=workflow_subtask,
project_base_path=project.base_path,
project_name=project.dirname,
project_id=project.project_id,
sample_id=sample.name)
if level == "sample":
if not keep_existing_data:
remove_previous_sample_analyses(project, sample)
default_files_to_copy=None
elif level == "genotype":
if not keep_existing_data:
remove_previous_genotype_analyses(project)
default_files_to_copy=None
# Update the project to keep only valid fastq files for setup.xml creation
if level == "genotype":
updated_project, default_files_to_copy = \
collect_files_for_sample_analysis(project,
sample,
restart_finished_jobs=True,
status_field="genotype_status")
else:
updated_project, default_files_to_copy = \
collect_files_for_sample_analysis(project,
sample,
restart_finished_jobs,
status_field="alignment_status")
setup_xml_cl, setup_xml_path = build_setup_xml(project=updated_project,
sample=sample,
workflow=workflow_subtask,
local_scratch_mode=(exec_mode == "sbatch"),
config=config)
piper_cl = build_piper_cl(project=project,
workflow_name=workflow_subtask,
setup_xml_path=setup_xml_path,
exit_code_path=exit_code_path,
config=config,
exec_mode=exec_mode,
generate_bqsr_bam=generate_bqsr_bam)
if exec_mode == "sbatch":
process_id = None
slurm_job_id = sbatch_piper_sample([setup_xml_cl, piper_cl],
workflow_subtask,
project, sample,
restart_finished_jobs=restart_finished_jobs,
files_to_copy=default_files_to_copy)
for x in xrange(10):
# Time delay to let sbatch get its act together
# (takes a few seconds to be visible with sacct)
try:
get_slurm_job_status(slurm_job_id)
break
except ValueError:
time.sleep(2)
else:
LOG.error('sbatch file for sample {}/{} did not '
'queue properly! Job ID {} cannot be '
'found.'.format(project, sample, slurm_job_id))
else: # "local"
raise NotImplementedError('Local execution not currently implemented. '
'I\'m sure Denis can help you with this.')
#slurm_job_id = None
#launch_piper_job(setup_xml_cl, project)
#process_handle = launch_piper_job(piper_cl, project)
#process_id = process_handle.pid
try:
record_process_sample(project=project,
sample=sample,
analysis_module_name="piper_ngi",
slurm_job_id=slurm_job_id,
process_id=process_id,
workflow_subtask=workflow_subtask)
except RuntimeError as e:
LOG.error(e)
## Question: should we just kill the run in this case or let it go?
continue
except (NotImplementedError, RuntimeError, ValueError) as e:
error_msg = ('Processing project "{}" / sample "{}" / workflow "{}" '
'failed: {}'.format(project, sample,
workflow_subtask,
e))
LOG.error(error_msg)
def update_charon_with_local_jobs_status(quiet=False, config=None, config_file_path=None):
"""Check the status of all locally-tracked jobs and update Charon accordingly.
"""
if quiet and not config.get("quiet"):
config['quiet'] = True
LOG.info("Updating Charon with the status of all locally-tracked jobs...")
multiqc_projects=set()
with get_db_session() as session:
charon_session = CharonSession()
for sample_entry in session.query(SampleAnalysis).all():
# Local names
workflow = sample_entry.workflow
project_name = sample_entry.project_name
project_id = sample_entry.project_id
project_base_path = sample_entry.project_base_path
sample_id = sample_entry.sample_id
engine = sample_entry.engine
# Only one of these id fields (slurm, pid) will have a value
slurm_job_id = sample_entry.slurm_job_id
process_id = sample_entry.process_id
piper_exit_code = get_exit_code(workflow_name=workflow,
project_base_path=project_base_path,
project_name=project_name,
project_id=project_id,
sample_id=sample_id)
label = "project/sample {}/{}".format(project_name, sample_id)
if workflow not in ("merge_process_variantcall", "genotype_concordance",):
LOG.error('Unknown workflow "{}" for {}; cannot update '
'Charon. Skipping sample.'.format(workflow, label))
continue
try:
project_obj = create_project_obj_from_analysis_log(project_name,
project_id,
project_base_path,
sample_id,
workflow)
except IOError as e: # analysis log file is missing!
error_text = ('Could not find analysis log file! Cannot update '
'Charon for {} run {}/{}: {}'.format(workflow,
project_id,
sample_id,
e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="ERROR",
info_text=error_text,
workflow=workflow)
continue
try:
if piper_exit_code == 0:
# 0 -> Job finished successfully
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
set_status = "ANALYZED" # sample level
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
set_status = "DONE" # sample level
recurse_status = "DONE" # For the seqrun level
info_text = ('Workflow "{}" for {} finished succesfully. '
'Recording status {} in Charon'.format(workflow,
label,
set_status))
LOG.info(info_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="INFO",
info_text=info_text,
workflow=workflow)
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=recurse_status,
config=config)
# Job is only deleted if the Charon status update succeeds
session.delete(sample_entry)
#add project to MultiQC
multiqc_projects.add((project_base_path, project_id, project_name))
if workflow == "merge_process_variantcall":
# Parse seqrun output results / update Charon
# This is a semi-optional step -- failure here will send an
# email but not more than once. The record is still removed
# from the local jobs database, so this will have to be done
# manually if you want it done at all.
piper_qc_dir = os.path.join(project_base_path, "ANALYSIS",
project_id, "piper_ngi",
"02_preliminary_alignment_qc")
update_coverage_for_sample_seqruns(project_id, sample_id,
piper_qc_dir)
update_sample_duplication_and_coverage(project_id, sample_id,
project_base_path)
elif workflow == "genotype_concordance":
piper_gt_dir = os.path.join(project_base_path, "ANALYSIS",
project_id, "piper_ngi",
"03_genotype_concordance")
try:
update_gtc_for_sample(project_id, sample_id, piper_gt_dir)
except (CharonError, IOError, ValueError) as e:
LOG.error(e)
elif type(piper_exit_code) is int and piper_exit_code > 0:
# 1 -> Job failed
set_status = "FAILED"
error_text = ('Workflow "{}" for {} failed. Recording status '
'{} in Charon.'.format(workflow, label, set_status))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine,
level="ERROR",
info_text=error_text,
workflow=workflow)
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj, status_field=seqrun_status_field,
status_value=set_status, config=config)
# Job is only deleted if the Charon update succeeds
session.delete(sample_entry)
else:
# None -> Job still running OR exit code was never written (failure)
JOB_FAILED = None
if slurm_job_id:
try:
slurm_exit_code = get_slurm_job_status(slurm_job_id)
except ValueError as e:
slurm_exit_code = 1
if slurm_exit_code is not None: # "None" indicates job is still running
JOB_FAILED = True
else:
if not psutil.pid_exists(process_id):
# Job did not write an exit code and is also not running
JOB_FAILED = True
if JOB_FAILED:
set_status = "FAILED"
error_text = ('No exit code found but job not running '
'for {} / {}: setting status to {} in '
'Charon'.format(label, workflow, set_status))
if slurm_job_id:
exit_code_file_path = \
create_exit_code_file_path(workflow_subtask=workflow,
project_base_path=project_base_path,
project_name=project_name,
project_id=project_id,
sample_id=sample_id)
error_text += (' (slurm job id "{}", exit code file path '
'"{}")'.format(slurm_job_id, exit_code_file_path))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name,
sample_name=sample_id,
engine_name=engine, level="ERROR",
info_text=error_text,
workflow=workflow)
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=set_status,
config=config)
# Job is only deleted if the Charon update succeeds
LOG.debug("Deleting local entry {}".format(sample_entry))
session.delete(sample_entry)
else: # Job still running
set_status = "UNDER_ANALYSIS"
if workflow == "merge_process_variantcall":
sample_status_field = "analysis_status"
seqrun_status_field = "alignment_status"
recurse_status = "RUNNING"
elif workflow == "genotype_concordance":
sample_status_field = seqrun_status_field = "genotype_status"
recurse_status = "UNDER_ANALYSIS"
try:
remote_sample=charon_session.sample_get(projectid=project_id, sampleid=sample_id)
charon_status = remote_sample.get(sample_status_field)
if charon_status and not charon_status == set_status:
LOG.warning('Tracking inconsistency for {}: Charon status '
'for field "{}" is "{}" but local process tracking '
'database indicates it is running. Setting value '
'in Charon to {}.'.format(label, sample_status_field,
charon_status, set_status))
charon_session.sample_update(projectid=project_id,
sampleid=sample_id,
**{sample_status_field: set_status})
recurse_status_for_sample(project_obj,
status_field=seqrun_status_field,
status_value=recurse_status,
config=config)
except CharonError as e:
error_text = ('Unable to update/verify Charon '
'for {}: {}'.format(label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
except CharonError as e:
error_text = ('Unable to update Charon for {}: '
'{}'.format(label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
except OSError as e:
error_text = ('Permissions error when trying to update Charon '
'"{}" status for "{}": {}'.format(workflow, label, e))
LOG.error(error_text)
if not config.get('quiet'):
mail_analysis(project_name=project_name, sample_name=sample_id,
engine_name=engine, level="ERROR",
workflow=workflow, info_text=error_text)
session.commit()
#Run Multiqc
for pj_tuple in multiqc_projects:
LOG.info("Running MultiQC on project {}".format(pj_tuple[1]))
run_multiqc(pj_tuple[0], pj_tuple[1], pj_tuple[2])