def test_return_cls_for_workflow(self, mock_qc):
config = {'A': 'B'}
mock_qc.return_value = ['command', '--option']
workflow_name = 'qc'
input_files = ['file', 'another_file']
output_dir = 'output'
command_line_list = workflows.return_cls_for_workflow(workflow_name,
input_files,
output_dir,
config=config)
self.assertEqual(command_line_list, [['command', '--option']])
mock_qc.assert_called_once_with(input_files, output_dir, config)
with self.assertRaises(NotImplementedError):
missing_workflow_name = 'something'
command_line_list = workflows.return_cls_for_workflow(
missing_workflow_name, input_files, output_dir, config=config)
def analyze(project, sample, quiet=False, config=None, config_file_path=None):
"""The main entry point for the qc pipeline."""
## TODO implement "quiet" feature
## TODO implement mailing on failure
LOG.info("Launching qc analysis for project/sample {}/{}".format(
project, sample))
project_analysis_path = os.path.join(project.base_path, "ANALYSIS",
project.project_id, "qc_ngi")
sample_analysis_path = os.path.join(project_analysis_path, sample.name)
log_dir_path = os.path.join(project_analysis_path, "logs")
safe_makedir(sample_analysis_path)
safe_makedir(log_dir_path)
fastq_files_to_process = []
src_fastq_base = os.path.join(project.base_path, "DATA",
project.project_id, sample.name)
for libprep in sample:
for seqrun in libprep:
for fastq_file in seqrun:
path_to_src_fastq = os.path.join(src_fastq_base, libprep.name,
seqrun.name, fastq_file)
fastq_files_to_process.append(path_to_src_fastq)
paired_fastq_files = list(
find_fastq_read_pairs(fastq_files_to_process).values())
qc_cl_list = return_cls_for_workflow("qc", paired_fastq_files,
sample_analysis_path)
sbatch_file_path = create_sbatch_file(qc_cl_list, project, sample, config)
try:
slurm_job_id = queue_sbatch_file(sbatch_file_path)
except RuntimeError as e:
LOG.error('Failed to queue qc sbatch file for project/sample '
'"{}"/"{}"!'.format(project, sample))
else:
LOG.info('Queued qc sbatch file for project/sample '
'"{}"/"{}": slurm job id {}'.format(project, sample,
slurm_job_id))
slurm_jobid_file = os.path.join(
log_dir_path, "{}-{}.slurmjobid".format(project.project_id,
sample))
LOG.info('Writing slurm job id "{}" to file "{}"'.format(
slurm_job_id, slurm_jobid_file))
try:
with open(slurm_jobid_file, 'w') as f:
f.write("{}\n".format(slurm_job_id))
except IOError as e:
LOG.warning('Could not write slurm job id for project/sample '
'{}/{} to file "{}" ({})'.format(
project, sample, slurm_jobid_file, e))
def analyze(project, sample, config=None, config_file_path=None):
"""The main entry point for the qc pipeline."""
LOG.info("Processing project/sample {}/{}".format(project, sample))
# Two paths diverged in a yellow wood
project_analysis_path = os.path.join(project.base_path,
"ANALYSIS",
project.project_id,
"qc_ngi")
# and sorry I could not travel both
sample_analysis_path = os.path.join(project_analysis_path, sample.name)
# and be one traveler, long I stood
log_dir_path = os.path.join(project_analysis_path, "logs")
# and looked down one as far as I could
safe_makedir(sample_analysis_path)
# To where it bent in the undergrowth
safe_makedir(log_dir_path)
# I need to go to sleep
fastq_files_to_process = []
# I suppose I -should- have quoted the other one
src_fastq_base = os.path.join(project.base_path, "DATA",
project.project_id, sample.name)
# Whose woods these are I think I know
for libprep in sample:
# His house is in the village though
for seqrun in libprep:
# He will not see mt stopping here
for fastq_file in seqrun:
# To watch
path_to_src_fastq = os.path.join(src_fastq_base,
libprep.name,
seqrun.name,
fastq_file)
# his woods
fastq_files_to_process.append(path_to_src_fastq)
# fill up
paired_fastq_files = find_fastq_read_pairs(fastq_files_to_process).values()
# with snow
qc_cl_list = return_cls_for_workflow("qc", paired_fastq_files, sample_analysis_path)
sbatch_file_path = create_sbatch_file(qc_cl_list, project, sample, config)
try:
slurm_job_id = queue_sbatch_file(sbatch_file_path)
except RuntimeError as e:
LOG.error('Failed to queue qc sbatch file for project/sample '
'"{}"/"{}"!'.format(project, sample))
else:
LOG.info('Queued qc sbatch file for project/sample '
'"{}"/"{}": slurm job id {}'.format(project, sample, slurm_job_id))
def analyze(project, sample, quiet=False, config=None, config_file_path=None):
"""The main entry point for the qc pipeline."""
## TODO implement "quiet" feature
## TODO implement mailing on failure
LOG.info("Launching qc analysis for project/sample {}/{}".format(project, sample))
project_analysis_path = os.path.join(project.base_path,
"ANALYSIS",
project.project_id,
"qc_ngi")
sample_analysis_path = os.path.join(project_analysis_path, sample.name)
log_dir_path = os.path.join(project_analysis_path, "logs")
safe_makedir(sample_analysis_path)
safe_makedir(log_dir_path)
fastq_files_to_process = []
src_fastq_base = os.path.join(project.base_path, "DATA",
project.project_id, sample.name)
for libprep in sample:
for seqrun in libprep:
for fastq_file in seqrun:
path_to_src_fastq = os.path.join(src_fastq_base,
libprep.name,
seqrun.name,
fastq_file)
fastq_files_to_process.append(path_to_src_fastq)
paired_fastq_files = find_fastq_read_pairs(fastq_files_to_process).values()
qc_cl_list = return_cls_for_workflow("qc", paired_fastq_files, sample_analysis_path)
sbatch_file_path = create_sbatch_file(qc_cl_list, project, sample, config)
try:
slurm_job_id = queue_sbatch_file(sbatch_file_path)
except RuntimeError as e:
LOG.error('Failed to queue qc sbatch file for project/sample '
'"{}"/"{}"!'.format(project, sample))
else:
LOG.info('Queued qc sbatch file for project/sample '
'"{}"/"{}": slurm job id {}'.format(project, sample, slurm_job_id))
slurm_jobid_file = os.path.join(log_dir_path,
"{}-{}.slurmjobid".format(project.project_id,
sample))
LOG.info('Writing slurm job id "{}" to file "{}"'.format(slurm_job_id,
slurm_jobid_file))
try:
with open(slurm_jobid_file, 'w') as f:
f.write("{}\n".format(slurm_job_id))
except IOError as e:
LOG.warn('Could not write slurm job id for project/sample '
'{}/{} to file "{}" ({}). So... yup. Good luck bro!'.format(e))
def analyze(project, sample, config=None, config_file_path=None):
"""The main entry point for the qc pipeline."""
LOG.info("Processing project/sample {}/{}".format(project, sample))
# Two paths diverged in a yellow wood
project_analysis_path = os.path.join(project.base_path, "ANALYSIS",
project.project_id, "qc_ngi")
# and sorry I could not travel both
sample_analysis_path = os.path.join(project_analysis_path, sample.name)
# and be one traveler, long I stood
log_dir_path = os.path.join(project_analysis_path, "logs")
# and looked down one as far as I could
safe_makedir(sample_analysis_path)
# To where it bent in the undergrowth
safe_makedir(log_dir_path)
# I need to go to sleep
fastq_files_to_process = []
# I suppose I -should- have quoted the other one
src_fastq_base = os.path.join(project.base_path, "DATA",
project.project_id, sample.name)
# Whose woods these are I think I know
for libprep in sample:
# His house is in the village though
for seqrun in libprep:
# He will not see mt stopping here
for fastq_file in seqrun:
# To watch
path_to_src_fastq = os.path.join(src_fastq_base, libprep.name,
seqrun.name, fastq_file)
# his woods
fastq_files_to_process.append(path_to_src_fastq)
# fill up
paired_fastq_files = find_fastq_read_pairs(fastq_files_to_process).values()
# with snow
qc_cl_list = return_cls_for_workflow("qc", paired_fastq_files,
sample_analysis_path)
sbatch_file_path = create_sbatch_file(qc_cl_list, project, sample, config)
try:
slurm_job_id = queue_sbatch_file(sbatch_file_path)
except RuntimeError as e:
LOG.error('Failed to queue qc sbatch file for project/sample '
'"{}"/"{}"!'.format(project, sample))
else:
LOG.info('Queued qc sbatch file for project/sample '
'"{}"/"{}": slurm job id {}'.format(project, sample,
slurm_job_id))
