def aneufinder_workflow(workflow, args): config = helpers.load_config(args) cellids = helpers.get_samples(args['input_yaml']) bam_files, _ = helpers.get_bams(args['input_yaml']) workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=cellids, ) info_file = os.path.join(args["out_dir"],'results', 'aneufinder', "info.yaml") output = os.path.join(args['out_dir'], 'results', "aneufinder") aneufinder_pdf_file = os.path.join( output, 'plots', '{}_reads.pdf'.format(args['library_id'])) helpers.makedirs(output) results_filename = os.path.join(output, '{}_results.h5'.format(args['library_id'])) workflow.subworkflow( name='aneufinder_workflow', func=aneufinder.create_aneufinder_workflow, args=( mgd.InputFile('bam_markdups', 'cell_id', fnames=bam_files), cellids, config, output, mgd.OutputFile(results_filename), mgd.OutputFile(aneufinder_pdf_file), args['library_id'], ), ) results = { 'aneufinder_plot': helpers.format_file_yaml(aneufinder_pdf_file), 'aneufinder_data':helpers.format_file_yaml(results_filename), } input_datasets = {k: helpers.format_file_yaml(v) for k,v in bam_files.iteritems()} metadata = { 'aneufinder':{ 'reads_table': '/aneufinder/reads', 'segments_table': '/aneufinder/segments/', 'chromosomes': config['chromosomes'], 'ref_genome': config['ref_genome'], 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform( name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], mem_retry_increment=2, ncpus=1), func="single_cell.utils.helpers.write_to_yaml", args=( mgd.OutputFile(info_file), metadata ) ) return workflow
def germline_calling_workflow(workflow, args): config = helpers.load_config(args) ctx = { 'mem_retry_increment': 2, 'ncpus': 1, 'mem': config["memory"]['low'], 'pool_id': config['pools']['standard'], } docker_ctx = helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') ctx.update(docker_ctx) bam_files, bai_files = helpers.get_bams(args['input_yaml']) sampleids = helpers.get_samples(args['input_yaml']) normal_bam_template = args["input_template"] normal_bai_template = args["input_template"] + ".bai" if "{reads}" in normal_bam_template: raise ValueError( "input template for germline calling only support region based splits" ) varcalls_dir = os.path.join(args['out_dir'], 'results', 'germline_calling') samtools_germline_vcf = os.path.join(varcalls_dir, 'raw', 'samtools_germline.vcf.gz') snpeff_vcf_filename = os.path.join(varcalls_dir, 'snpeff.vcf') normal_genotype_filename = os.path.join(varcalls_dir, 'raw', 'normal_genotype.h5') mappability_filename = os.path.join(varcalls_dir, 'raw', 'mappability.h5') counts_template = os.path.join(varcalls_dir, 'counts', 'raw', 'counts.h5') germline_h5_filename = os.path.join(varcalls_dir, 'germline.h5') workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=bam_files.keys(), ) workflow.transform( name="get_regions", ctx=ctx, func="single_cell.utils.pysamutils.get_regions_from_reference", ret=pypeliner.managed.OutputChunks('region'), args=( config["ref_genome"], config["split_size"], config["chromosomes"], )) workflow.subworkflow(name='samtools_germline', func=germline.create_samtools_germline_workflow, args=( mgd.InputFile("normal.split.bam", "region", template=normal_bam_template), mgd.InputFile("normal.split.bam.bai", "region", template=normal_bai_template), config['ref_genome'], mgd.OutputFile(samtools_germline_vcf, extensions=['.tbi']), config, ), kwargs={ 'chromosomes': config["chromosomes"], 'base_docker': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline'), 'vcftools_docker': helpers.get_container_ctx(config['containers'], 'vcftools'), 'samtools_docker': helpers.get_container_ctx(config['containers'], 'samtools'), }) workflow.subworkflow( name='annotate_mappability', func= "biowrappers.components.variant_calling.mappability.create_vcf_mappability_annotation_workflow", args=( config['databases']['mappability']['local_path'], mgd.InputFile(samtools_germline_vcf, extensions=['.tbi']), mgd.OutputFile(mappability_filename), ), kwargs={ 'base_docker': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') }) workflow.transform( name='annotate_genotype', func="single_cell.workflows.germline.tasks.annotate_normal_genotype", ctx=ctx, args=( mgd.InputFile(samtools_germline_vcf, extensions=['.tbi']), mgd.OutputFile(normal_genotype_filename), config["chromosomes"], ), ) workflow.subworkflow( name='snpeff', func= "biowrappers.components.variant_calling.snpeff.create_snpeff_annotation_workflow", args=( config['databases']['snpeff']['db'], mgd.InputFile(samtools_germline_vcf, extensions=['.tbi']), mgd.OutputFile(snpeff_vcf_filename), ), kwargs={ 'hdf5_output': False, 'base_docker': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline'), 'vcftools_docker': helpers.get_container_ctx(config['containers'], 'vcftools'), 'snpeff_docker': helpers.get_container_ctx(config['containers'], 'snpeff'), }) workflow.subworkflow( name='read_counts', func= "single_cell.variant_calling.create_snv_allele_counts_for_vcf_targets_workflow", args=( config, mgd.InputFile('tumour.bam', 'cell_id', fnames=bam_files), mgd.InputFile('tumour.bam.bai', 'cell_id', fnames=bai_files), mgd.InputFile(samtools_germline_vcf, extensions=['.tbi']), mgd.OutputFile(counts_template), ), kwargs={ 'table_name': '/germline_allele_counts', 'docker_config': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') }, ) workflow.transform( name='build_results_file', func="biowrappers.components.io.hdf5.tasks.concatenate_tables", ctx=ctx, args=( [ mgd.InputFile(counts_template), mgd.InputFile(mappability_filename), mgd.InputFile(normal_genotype_filename), ], pypeliner.managed.OutputFile(germline_h5_filename), ), kwargs={ 'drop_duplicates': True, }) info_file = os.path.join(args["out_dir"], 'results', 'germline_calling', "info.yaml") results = { 'germline_data': helpers.format_file_yaml(germline_h5_filename), } input_datasets = { k: helpers.format_file_yaml(v) for k, v in bam_files.iteritems() } metadata = { 'germline_calling': { 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], mem_retry_increment=2, ncpus=1), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def align_workflow(workflow, args): config = helpers.load_config(args) sampleinfo = helpers.get_sample_info(args['input_yaml']) cellids = helpers.get_samples(args['input_yaml']) bam_files, bai_files = helpers.get_bams(args['input_yaml']) lib = args["library_id"] outdir = os.path.join(args["out_dir"], "results", "alignment") info_file = os.path.join(outdir, "info.yaml") alignment_metrics_h5 = os.path.join(outdir, '{}_alignment_metrics.h5'.format(lib)) plots_dir = os.path.join(outdir, 'plots') plot_metrics_output = os.path.join(plots_dir, '{}_plot_metrics.pdf'.format(lib)) ctx = {'mem_retry_increment': 2, 'ncpus': 1} ctx.update( helpers.get_container_ctx(config['containers'], 'single_cell_pipeline')) if not args["metrics_only"]: fastq1_files, fastq2_files = helpers.get_fastqs(args['input_yaml']) instrumentinfo = helpers.get_instrument_info(args['input_yaml']) centerinfo = helpers.get_center_info(args['input_yaml']) workflow.setobj( obj=mgd.OutputChunks('cell_id', 'lane'), value=fastq1_files.keys(), ) workflow.subworkflow( name='alignment_workflow', func=align.create_alignment_workflow, args=( mgd.InputFile('fastq_1', 'cell_id', 'lane', fnames=fastq1_files, axes_origin=[]), mgd.InputFile('fastq_2', 'cell_id', 'lane', fnames=fastq2_files, axes_origin=[]), mgd.OutputFile('bam_markdups', 'cell_id', fnames=bam_files, axes_origin=[]), mgd.OutputFile('bai_markdups', 'cell_id', fnames=bai_files, axes_origin=[]), config['ref_genome'], config, args, instrumentinfo, centerinfo, sampleinfo, cellids, ), ) else: workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=cellids, ) workflow.subworkflow( name='metrics_workflow', func=alignment_metrics.create_alignment_metrics_workflow, args=( mgd.InputFile('bam_markdups', 'cell_id', fnames=bam_files, axes_origin=[]), mgd.InputFile('bai_markdups', 'cell_id', fnames=bai_files, axes_origin=[]), mgd.OutputFile(alignment_metrics_h5), mgd.OutputFile(plot_metrics_output), config['ref_genome'], config, args, sampleinfo, cellids, ), ) inputs = helpers.get_fastq_files(args["input_yaml"]) outputs = { k: helpers.format_file_yaml(v) for k, v in bam_files.iteritems() } metadata = { 'alignment': { 'name': 'alignment', 'cell_batch_realign': args["realign"], 'metrics_table': '/alignment/metrics', 'gc_metrics_table': '/alignment/gc_metrics', 'aligner': config["aligner"], 'adapter': config["adapter"], 'adapter2': config["adapter2"], 'picardtools_wgsmetrics_params': config['picard_wgs_params'], 'ref_genome': config["ref_genome"], 'version': single_cell.__version__, 'containers': config['containers'], 'output_datasets': outputs, 'input_datasets': inputs, 'results': { 'alignment_metrics': helpers.format_file_yaml(alignment_metrics_h5), 'alignment_plots': helpers.format_file_yaml(plot_metrics_output), }, } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], **ctx), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def create_variant_calling_workflow( tumour_cell_bams, tumour_region_bams, normal_region_bams, museq_vcf, strelka_snv_vcf, strelka_indel_vcf, snv_h5, config, raw_data_dir, ): workflow = pypeliner.workflow.Workflow() workflow.set_filenames('normal_regions.bam', 'region', fnames=normal_region_bams) workflow.set_filenames('tumour_cells.bam', 'cell_id', fnames=tumour_cell_bams) workflow.set_filenames('tumour_regions.bam', 'region', fnames=tumour_region_bams) workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=tumour_cell_bams.keys(), ) workflow.setobj( obj=mgd.OutputChunks('region'), value=tumour_region_bams.keys(), ) workflow.subworkflow( name='museq', func=mutationseq.create_museq_workflow, args=( mgd.InputFile('normal_regions.bam', 'region', extensions=['.bai']), mgd.InputFile('tumour_regions.bam', 'region', extensions=['.bai']), config['ref_genome'], mgd.OutputFile(museq_vcf), config, ), ) workflow.subworkflow(name='strelka', func=strelka.create_strelka_workflow, args=( mgd.InputFile('normal_regions.bam', 'region', extensions=['.bai']), mgd.InputFile('tumour_regions.bam', 'region', extensions=['.bai']), config['ref_genome'], mgd.OutputFile(strelka_indel_vcf), mgd.OutputFile(strelka_snv_vcf), config, ), kwargs={"chromosomes": config["chromosomes"]}) workflow.transform( name='convert_museq_to_hdf5', func="biowrappers.components.io.vcf.tasks.convert_vcf_to_hdf5", ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), args=( mgd.InputFile(museq_vcf), mgd.TempOutputFile('museq.h5'), '/museq/vcf/', ), kwargs={ 'score_callback': museq_callback, }) workflow.transform( name='convert_strelka_to_hdf5', func="biowrappers.components.io.vcf.tasks.convert_vcf_to_hdf5", ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), args=( mgd.InputFile(strelka_snv_vcf), mgd.TempOutputFile('strelka_snv.h5'), '/strelka/vcf/', ), kwargs={ 'score_callback': strelka_snv_callback, }) workflow.transform(name='merge_snvs', func='biowrappers.components.io.vcf.tasks.merge_vcfs', ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), args=([ mgd.InputFile(museq_vcf), mgd.InputFile(strelka_snv_vcf), ], mgd.TempOutputFile('all.snv.vcf')), kwargs={ 'docker_config': helpers.get_container_ctx(config['containers'], 'vcftools') }) workflow.transform(name='finalise_snvs', func="biowrappers.components.io.vcf.tasks.finalise_vcf", ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), args=(mgd.TempInputFile('all.snv.vcf'), mgd.TempOutputFile('all.snv.vcf.gz', extensions=['.tbi'])), kwargs={ 'docker_config': helpers.get_container_ctx(config['containers'], 'vcftools') }) workflow.subworkflow( name='annotate_snvs', axes=(), func= "biowrappers.pipelines.snv_call_and_annotate.create_annotation_workflow", args=( config, mgd.TempInputFile('all.snv.vcf.gz'), mgd.TempOutputFile('snv_annotations.h5'), os.path.join(raw_data_dir, 'snv'), ), kwargs={ 'variant_type': 'snv', 'docker_config': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') }) workflow.subworkflow( name='count_alleles', func=create_snv_allele_counts_for_vcf_targets_workflow, args=( config, mgd.InputFile('tumour_cells.bam', 'cell_id', extensions=['.bai']), mgd.TempInputFile('all.snv.vcf.gz'), mgd.TempOutputFile('snv_counts.h5'), ), kwargs={ 'chromosomes': config['chromosomes'], 'docker_config': helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') }) workflow.transform( name='build_results_file', ctx=dict(mem=config['memory']['high'], pool_id=config['pools']['highmem'], **ctx), func="biowrappers.components.io.hdf5.tasks.concatenate_tables", args=( [ mgd.TempInputFile('snv_counts.h5'), mgd.TempInputFile('snv_annotations.h5'), mgd.TempInputFile('museq.h5'), mgd.TempInputFile('strelka_snv.h5'), ], pypeliner.managed.OutputFile(snv_h5), ), kwargs={ 'drop_duplicates': True, 'in_memory': False, }) info_file = os.path.join(args["out_dir"], 'results', 'variant_calling', "info.yaml") normals = { k: helpers.format_file_yaml(v) for k, v in normal_region_bams.iteritems() } tumours = { k: helpers.format_file_yaml(v) for k, v in tumour_region_bams.iteritems() } cells = { k: helpers.format_file_yaml(v) for k, v in tumour_cell_bams.iteritems() } inputs = {'normal': normals, 'tumour': tumours, 'cells': cells} metadata = { 'variant_calling': { 'name': 'variant_calling', 'version': single_cell.__version__, 'containers': config['containers'], 'output_datasets': None, 'input_datasets': inputs, 'results': { 'variant_calling_data': helpers.format_file_yaml(snv_h5) } } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard']), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def ltm_workflow(workflow, args): config = helpers.load_config(args) hmmcopy, timepoints = ltmutils.read_input_file(args['input_csv']) cn_matrix = os.path.join(args['out_dir'], 'cn_matrix.csv') output_gml = os.path.join(args['out_dir'], 'tree.gml') output_rooted_gml = os.path.join(args['out_dir'], 'rooted_tree.gml') # Outputs required for visualization with cellscape cnv_annots_csv = os.path.join(args['out_dir'], 'cnv_annots.csv') cnv_tree_edges_csv = os.path.join(args['out_dir'], 'cnv_tree_edges.csv') cnv_data_csv = os.path.join(args['out_dir'], 'cnv_data.csv') output_rmd = os.path.join(args['out_dir'], 'cellscape.Rmd') root_id_file = os.path.join(args['out_dir'], 'root_id.txt') workflow.setobj( obj=mgd.OutputChunks('timepoint'), value=timepoints, ) workflow.subworkflow( name='ltm_scale', func=ltm.create_ltm_workflow, args=( mgd.InputFile('hmmcopy.h5', 'timepoint', fnames=hmmcopy), mgd.OutputFile(cn_matrix), mgd.OutputFile(output_gml), mgd.OutputFile(output_rooted_gml), mgd.OutputFile(cnv_annots_csv), mgd.OutputFile(cnv_tree_edges_csv), mgd.OutputFile(cnv_data_csv), mgd.OutputFile(output_rmd), config, args['root_id'], mgd.OutputFile(root_id_file), args['number_of_jobs'], args['ploidy'], ), ) info_file = os.path.join(args["out_dir"], 'results', 'ltm', "info.yaml") results = { 'ltm_cn_matrix': helpers.format_file_yaml(cn_matrix), 'ltm_gml': helpers.format_file_yaml(output_gml), 'ltm_rooted_gml': helpers.format_file_yaml(output_rooted_gml), 'ltm_cnv_annots_csv': helpers.format_file_yaml(cnv_annots_csv), 'ltm_cnv_tree_edges_csv': helpers.format_file_yaml(cnv_tree_edges_csv), 'ltm_cnv_data_csv': helpers.format_file_yaml(cnv_data_csv), 'ltm_output_rmd': helpers.format_file_yaml(output_rmd) } input_datasets = { k: helpers.format_file_yaml(v) for k, v in bam_file.iteritems() } metadata = { 'LTM': { 'chromosomes': config['chromosomes'], 'ref_genome': config['ref_genome'], 'cell_filters': config["good_cells"], 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform(name='generate_meta_yaml', ctx=dict( mem=config['memory']['med'], pool_id=config['pools']['standard'], ), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def breakpoint_calling_workflow(workflow, args): config = helpers.load_config(args) normal_bam_file = args['matched_normal'] bam_files, bai_files = helpers.get_bams(args['input_yaml']) varcalls_dir = os.path.join(args['out_dir'], 'results', 'breakpoint_calling') raw_data_directory = os.path.join(varcalls_dir, 'raw') breakpoints_filename = os.path.join(varcalls_dir, 'breakpoints.h5') ref_data_directory = '/refdata' pypeliner.workflow.Workflow() workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=bam_files.keys(), ) workflow.subworkflow( name='destruct', func= "biowrappers.components.breakpoint_calling.destruct.destruct_pipeline", args=( mgd.InputFile(normal_bam_file), mgd.InputFile('tumour.bam', 'cell_id', fnames=bam_files), config.get('destruct', {}), ref_data_directory, mgd.OutputFile(breakpoints_filename), raw_data_directory, ), ) info_file = os.path.join(args["out_dir"], 'results', 'breakpoint_calling', "info.yaml") results = { 'destruct_data': helpers.format_file_yaml(breakpoints_filename), } input_datasets = { k: helpers.format_file_yaml(v) for k, v in bam_files.iteritems() } input_datasets = {'normal': normal_bam_file, 'tumour': input_datasets} metadata = { 'breakpoint_calling': { 'ref_data': ref_data_directory, 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], mem_retry_increment=2, ncpus=1), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def copy_number_calling_workflow(workflow, args): config = helpers.load_config(args) ctx = {'mem_retry_increment': 2, 'ncpus': 1, 'mem': config["memory"]['low'], 'pool_id': config['pools']['standard']} docker_ctx = helpers.get_container_ctx(config['containers'], 'single_cell_pipeline') ctx.update(docker_ctx) tumour_bam_files, tumour_bai_files = helpers.get_bams(args['tumour_yaml']) normal_bam_files, normal_bai_files = helpers.get_bams(args['normal_yaml']) tumour_cellids = helpers.get_samples(args['tumour_yaml']) normal_cellids = helpers.get_samples(args['normal_yaml']) if set(tumour_bam_files.keys()) != set(tumour_cellids): raise ValueError() if set(normal_bam_files.keys()) != set(normal_cellids): raise ValueError() copynumber_dir = os.path.join(args["out_dir"], "copynumber") out_file = os.path.join(copynumber_dir, "results", "results.h5") cloneid = args["clone_id"] remixt_config = config['titan_params'].get('extract_seqdata', {}) workflow.setobj( obj=mgd.OutputChunks('tumour_cell_id'), value=tumour_cellids, ) workflow.setobj( obj=mgd.OutputChunks('normal_cell_id'), value=normal_cellids, ) workflow.transform( name="get_snp_positions_filename", ctx=ctx, func="remixt.config.get_filename", ret=mgd.TempOutputObj('snp_positions_filename'), args=( remixt_config, config['titan_params']['ref_data_dir'], 'snp_positions' ) ) workflow.transform( name="get_bam_max_fragment_length", ctx=ctx, func="remixt.config.get_param", ret=mgd.TempOutputObj('bam_max_fragment_length'), args=( remixt_config, 'bam_max_fragment_length' ) ) workflow.transform( name="get_bam_max_soft_clipped", ctx=ctx, func="remixt.config.get_param", ret=mgd.TempOutputObj('bam_max_soft_clipped'), args=( remixt_config, 'bam_max_soft_clipped' ) ) workflow.transform( name="get_bam_check_proper_pair", ctx=ctx, func="remixt.config.get_param", ret=mgd.TempOutputObj('bam_check_proper_pair'), args=( remixt_config, 'bam_check_proper_pair' ) ) workflow.subworkflow( name="extract_seqdata_tumour", axes=('tumour_cell_id',), func=extract_seqdata.create_extract_seqdata_workflow, args=( mgd.InputFile( 'bam_markdups', 'tumour_cell_id', fnames=tumour_bam_files), mgd.InputFile( 'bam_markdups_index', 'tumour_cell_id', fnames=tumour_bai_files), mgd.TempOutputFile("tumour.h5", "tumour_cell_id"), config, config['titan_params'].get('extract_seqdata', {}), config['titan_params']['ref_data_dir'], mgd.TempInputObj('snp_positions_filename'), mgd.TempInputObj('bam_max_fragment_length'), mgd.TempInputObj('bam_max_soft_clipped'), mgd.TempInputObj('bam_check_proper_pair'), ) ) workflow.subworkflow( name="extract_seqdata_normal", axes=('normal_cell_id',), func=extract_seqdata.create_extract_seqdata_workflow, args=( mgd.InputFile( 'bam_markdups', 'normal_cell_id', fnames=normal_bam_files), mgd.InputFile( 'bam_markdups_index', 'normal_cell_id', fnames=normal_bai_files), mgd.TempOutputFile("normal.h5", "normal_cell_id"), config, config['titan_params'].get('extract_seqdata', {}), config['titan_params']['ref_data_dir'], mgd.TempInputObj('snp_positions_filename'), mgd.TempInputObj('bam_max_fragment_length'), mgd.TempInputObj('bam_max_soft_clipped'), mgd.TempInputObj('bam_check_proper_pair'), ) ) workflow.subworkflow( name='titan_workflow', func=titan.create_titan_workflow, args=( mgd.TempInputFile("normal.h5", "normal_cell_id"), mgd.TempInputFile("tumour.h5", "tumour_cell_id"), config['ref_genome'], copynumber_dir, out_file, config, args, tumour_cellids, normal_cellids, cloneid ), ) info_file = os.path.join(args["out_dir"],'results','copynumber_calling', "info.yaml") results = { 'copynumber_data': helpers.format_file_yaml(out_file), } tumours = {k: helpers.format_file_yaml(v) for k,v in tumour_bam_files.iteritems()} normals = {k: helpers.format_file_yaml(v) for k,v in normal_bam_files.iteritems()} input_datasets = {'tumour': tumours, 'normal': normals} metadata = { 'copynumber_calling': { 'chromosomes': config['chromosomes'], 'ref_genome': config['ref_genome'], 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform( name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], mem_retry_increment=2, ncpus=1), func="single_cell.utils.helpers.write_to_yaml", args=( mgd.OutputFile(info_file), metadata ) ) return workflow
def merge_bams_workflow(workflow, args): input_yaml = args["input_yaml"] output_template = args["merged_bam_template"] info_file = os.path.join(args["out_dir"], 'results', 'merge_bams', "info.yaml") config = helpers.load_config(args) bam_files, bai_files = helpers.get_bams(input_yaml) cellids = helpers.get_samples(input_yaml) wgs_bam_template = output_template wgs_bai_template = wgs_bam_template + ".bai" ctx = {'mem_retry_increment': 2, 'ncpus': 1} ctx.update( helpers.get_container_ctx(config['containers'], 'single_cell_pipeline')) workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=cellids, ) workflow.transform( name="get_regions", ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), func="single_cell.utils.pysamutils.get_regions_from_reference", ret=pypeliner.managed.TempOutputObj('region'), args=( config["ref_genome"], config["split_size"], config["chromosomes"], )) workflow.subworkflow(name="wgs_merge_workflow", func=merge_bams.create_merge_bams_workflow, args=( mgd.InputFile('bam_markdups', 'cell_id', fnames=bam_files, extensions=['.bai']), mgd.OutputFile("merged_bam", "region", axes_origin=[], template=wgs_bam_template, extensions=['.bai']), cellids, config, mgd.TempInputObj("region"), )) workflow.transform(name="get_files", ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), func='single_cell.utils.helpers.resolve_template', ret=pypeliner.managed.TempOutputObj('outputs'), args=(pypeliner.managed.TempInputObj('region'), wgs_bam_template, 'region')) inputs = {k: helpers.format_file_yaml(v) for k, v in bam_files.iteritems()} metadata = { 'merge_bams': { 'name': 'merge_bams', 'ref_genome': config["ref_genome"], 'version': single_cell.__version__, 'containers': config['containers'], 'output_datasets': pypeliner.managed.TempInputObj('outputs'), 'input_datasets': inputs, 'results': None } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=2, pool_id=config['pools']['standard'], **ctx), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow
def infer_haps_workflow(workflow, args): config = helpers.load_config(args) remixt_config = config['titan_params'].get('extract_seqdata', {}) singlecellimage = config['docker']['images']['single_cell_pipeline'] ctx = { 'mem_retry_increment': 2, 'ncpus': 1, 'image': singlecellimage['image'], 'dockerize': config['docker']['dockerize'], 'mounts': config['docker']['mounts'], 'username': singlecellimage['username'], 'password': singlecellimage['password'], 'server': singlecellimage['server'], } haps_dir = os.path.join(args["out_dir"], "infer_haps") haplotypes_filename = os.path.join(haps_dir, "results", "haplotypes.tsv") allele_counts_filename = os.path.join(haps_dir, "results", "allele_counts.tsv") snp_positions_filename = remixt.config.get_filename( config, ref_data_dir, 'snp_positions') bam_max_fragment_length = remixt.config.get_param( config, 'bam_max_fragment_length') bam_max_soft_clipped = remixt.config.get_param(config, 'bam_max_soft_clipped') bam_check_proper_pair = remixt.config.get_param(config, 'bam_check_proper_pair') workflow.setobj(obj=mgd.OutputChunks('chromosome'), value=config['titan_params']['chromosomes']) if args['input_yaml']: bam_files, bai_files = helpers.get_bams(args['input_yaml']) cellids = helpers.get_samples(args['input_yaml']) workflow.setobj( obj=mgd.OutputChunks('cell_id'), value=cellids, ) workflow.subworkflow( name="extract_seqdata", axes=('cell_id', ), func=extract_seqdata.create_extract_seqdata_workflow, args=( mgd.InputFile('bam_markdups', 'cell_id', fnames=bam_files), mgd.InputFile('bam_markdups_index', 'cell_id', fnames=bai_files), mgd.TempOutputFile("tumour.h5", "cell_id"), config, config['titan_params'].get('extract_seqdata', {}), config['titan_params']['ref_data_dir'], snp_positions_filename, bam_max_fragment_length, bam_max_soft_clipped, bam_check_proper_pair, )) workflow.transform( name='merge_all_seqdata', ctx=dict(mem=config["memory"]['high'], pool_id=config['pools']['highmem'], **ctx), func="single_cell.workflows.titan.tasks.merge_overlapping_seqdata", args=(mgd.TempOutputFile("seqdata_normal_all_cells_merged.h5"), mgd.TempInputFile("tumour.h5", "cell_id"), config["titan_params"]["chromosomes"]), ) else: workflow.subworkflow( name="extract_seqdata", func=extract_seqdata.create_extract_seqdata_workflow, args=( mgd.InputFile(args['input_bam']), mgd.InputFile(args['input_bam'] + '.bai'), mgd.TempOutputFile("seqdata_normal_all_cells_merged.h5"), config, config['titan_params'].get('extract_seqdata', {}), config['titan_params']['ref_data_dir'], snp_positions_filename, bam_max_fragment_length, bam_max_soft_clipped, bam_check_proper_pair, ), kwargs={'multiprocess': True}) if args["normal"]: workflow.transform( name='infer_snp_genotype', axes=('chromosome', ), ctx={'mem': 16}, func='remixt.analysis.haplotype.infer_snp_genotype_from_normal', args=( mgd.TempOutputFile('snp_genotype.tsv', 'chromosome'), mgd.TempInputFile("seqdata_normal_all_cells_merged.h5"), mgd.InputInstance('chromosome'), config, ), ) else: workflow.transform( name='infer_snp_genotype', axes=('chromosome', ), ctx={'mem': 16}, func='remixt.analysis.haplotype.infer_snp_genotype_from_tumour', args=( mgd.TempOutputFile('snp_genotype.tsv', 'chromosome'), { 'sample': mgd.TempInputFile("seqdata_normal_all_cells_merged.h5") }, mgd.InputInstance('chromosome'), config, ), ) workflow.transform(name='infer_haps', axes=('chromosome', ), ctx={'mem': 16}, func='remixt.analysis.haplotype.infer_haps', args=( mgd.TempOutputFile('haps.tsv', 'chromosome'), mgd.TempInputFile('snp_genotype.tsv', 'chromosome'), mgd.InputInstance('chromosome'), mgd.TempSpace('haplotyping', 'chromosome'), config, config['titan_params']['ref_data_dir'], )) workflow.transform(name='merge_haps', ctx={'mem': 16}, func='remixt.utils.merge_tables', args=( mgd.OutputFile(haplotypes_filename), mgd.TempInputFile('haps.tsv', 'chromosome'), )) workflow.transform( name='create_segments', func='remixt.analysis.segment.create_segments', args=( mgd.TempOutputFile('segments.tsv'), config, config['titan_params']['ref_data_dir'], ), ) workflow.transform( name='haplotype_allele_readcount', ctx={'mem': 20}, func='remixt.analysis.readcount.haplotype_allele_readcount', args=( mgd.OutputFile(allele_counts_filename), mgd.TempInputFile('segments.tsv'), mgd.TempInputFile('tumour.h5', 'cell_id'), mgd.InputFile(haplotypes_filename), config, ), ) info_file = os.path.join(args["out_dir"], 'results', 'infer_haps', "info.yaml") results = { 'infer_haps_allele_counts': helpers.format_file_yaml(allele_counts_filename), 'infer_haps_data': helpers.format_file_yaml(haplotypes_filename), } if args['input_yaml']: input_datasets = { k: helpers.format_file_yaml(v) for k, v in bam_files.iteritems() } else: input_datasets = helpers.format_file_yaml(args['input_bam']) metadata = { 'infer_haps': { 'version': single_cell.__version__, 'results': results, 'containers': config['containers'], 'input_datasets': input_datasets, 'output_datasets': None } } workflow.transform(name='generate_meta_yaml', ctx=dict(mem=config['memory']['med'], pool_id=config['pools']['standard'], mem_retry_increment=2, ncpus=1), func="single_cell.utils.helpers.write_to_yaml", args=(mgd.OutputFile(info_file), metadata)) return workflow