def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: make_examples does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() hts_verbose.set(hts_verbose.htsLogLevel[FLAGS.hts_logging_level]) # Set up options; may do I/O. options = default_options(add_flags=True, flags_obj=FLAGS) # Check arguments that apply to any mode. if not options.reference_filename: errors.log_and_raise('ref argument is required.', errors.CommandLineError) if not options.reads_filename: errors.log_and_raise('reads argument is required.', errors.CommandLineError) if not options.examples_filename: errors.log_and_raise('examples argument is required.', errors.CommandLineError) if options.n_cores != 1: errors.log_and_raise( 'Currently only supports n_cores == 1 but got {}.'.format( options.n_cores), errors.CommandLineError) # Check for argument issues specific to train mode. if in_training_mode(options): if not options.truth_variants_filename: errors.log_and_raise( 'truth_variants is required when in training mode.', errors.CommandLineError) if not options.confident_regions_filename: errors.log_and_raise( 'confident_regions is required when in training mode.', errors.CommandLineError) if options.gvcf_filename: errors.log_and_raise('gvcf is not allowed in training mode.', errors.CommandLineError) else: # Check for argument issues specific to calling mode. if options.variant_caller_options.sample_name == _UNKNOWN_SAMPLE: errors.log_and_raise('sample_name must be specified in calling mode.', errors.CommandLineError) if options.variant_caller_options.gq_resolution < 1: errors.log_and_raise('gq_resolution must be a non-negative integer.', errors.CommandLineError) # Run! make_examples_runner(options)
def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: call_variants does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() model = modeling.get_model(FLAGS.model_name) call_variants( examples_filename=FLAGS.examples, checkpoint_path=FLAGS.checkpoint, model=model, execution_hardware=FLAGS.execution_hardware, output_file=FLAGS.outfile, max_batches=FLAGS.max_batches, batch_size=FLAGS.batch_size)
def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: call_variants does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() model = modeling.get_model(FLAGS.model_name) call_variants( examples_filename=FLAGS.examples, checkpoint_path=FLAGS.checkpoint, model=model, execution_hardware=FLAGS.execution_hardware, output_file=FLAGS.outfile, max_batches=FLAGS.max_batches, batch_size=FLAGS.batch_size)
def test_clean_commandline_error_exit_clean_exit(self, exc_type, exit_value): with mock.patch.object(sys, 'exit') as mock_exit: with errors.clean_commandline_error_exit(exit_value=exit_value): raise exc_type() mock_exit.assert_called_once_with(exit_value)
def test_clean_commandline_error_exit_raise_non_allowed( self, exc_type, msg): with self.assertRaisesRegexp(exc_type, msg): with errors.clean_commandline_error_exit(): raise exc_type(msg)
def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: postprocess_variants does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. if (not FLAGS.nonvariant_site_tfrecord_path) != ( not FLAGS.gvcf_outfile): errors.log_and_raise( 'gVCF creation requires both nonvariant_site_tfrecord_path and ' 'gvcf_outfile flags to be set.', errors.CommandLineError) proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() fasta_reader = fasta.IndexedFastaReader(FLAGS.ref, cache_size=_FASTA_CACHE_SIZE) contigs = fasta_reader.header.contigs paths = io_utils.maybe_generate_sharded_filenames(FLAGS.infile) # Read one CallVariantsOutput record and extract the sample name from it. # Note that this assumes that all CallVariantsOutput protos in the infile # contain a single VariantCall within their constituent Variant proto, and # that the call_set_name is identical in each of the records. record = tf_utils.get_one_example_from_examples_path( ','.join(paths), proto=deepvariant_pb2.CallVariantsOutput) if record is None: raise ValueError('Cannot find any records in {}'.format( ','.join(paths))) sample_name = _extract_single_sample_name(record) header = dv_vcf_constants.deepvariant_header( contigs=contigs, sample_names=[sample_name]) with tempfile.NamedTemporaryFile() as temp: postprocess_variants_lib.process_single_sites_tfrecords( contigs, paths, temp.name) independent_variants = _transform_call_variants_output_to_variants( input_sorted_tfrecord_path=temp.name, qual_filter=FLAGS.qual_filter, multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter, sample_name=sample_name) variant_generator = haplotypes.maybe_resolve_conflicting_variants( independent_variants) write_variants_to_vcf(variant_generator=variant_generator, output_vcf_path=FLAGS.outfile, header=header) # Also write out the gVCF file if it was provided. if FLAGS.nonvariant_site_tfrecord_path: nonvariant_generator = io_utils.read_shard_sorted_tfrecords( FLAGS.nonvariant_site_tfrecord_path, key=_get_contig_based_variant_sort_keyfn(contigs), proto=variants_pb2.Variant) with vcf.VcfReader(FLAGS.outfile) as variant_reader: lessthanfn = _get_contig_based_lessthan(contigs) gvcf_variants = (_transform_to_gvcf_record(variant) for variant in variant_reader.iterate()) merged_variants = merge_variants_and_nonvariants( gvcf_variants, nonvariant_generator, lessthanfn, fasta_reader) write_variants_to_vcf(variant_generator=merged_variants, output_vcf_path=FLAGS.gvcf_outfile, header=header)
def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: postprocess_variants does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. if (not FLAGS.nonvariant_site_tfrecord_path) != (not FLAGS.gvcf_outfile): errors.log_and_raise( 'gVCF creation requires both nonvariant_site_tfrecord_path and ' 'gvcf_outfile flags to be set.', errors.CommandLineError) proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() fasta_reader = fasta.IndexedFastaReader( FLAGS.ref, cache_size=_FASTA_CACHE_SIZE) contigs = fasta_reader.header.contigs paths = sharded_file_utils.maybe_generate_sharded_filenames(FLAGS.infile) # Read one CallVariantsOutput record and extract the sample name from it. # Note that this assumes that all CallVariantsOutput protos in the infile # contain a single VariantCall within their constituent Variant proto, and # that the call_set_name is identical in each of the records. record = tf_utils.get_one_example_from_examples_path( ','.join(paths), proto=deepvariant_pb2.CallVariantsOutput) if record is None: logging.info('call_variants_output is empty. Writing out empty VCF.') sample_name = dv_constants.DEFAULT_SAMPLE_NAME if FLAGS.sample_name: logging.info( '--sample_name is set in postprocess_variant. Using %s as the ' 'sample name.', FLAGS.sample_name) sample_name = FLAGS.sample_name variant_generator = iter([]) else: sample_name = _extract_single_sample_name(record) temp = tempfile.NamedTemporaryFile() start_time = time.time() postprocess_variants_lib.process_single_sites_tfrecords( contigs, paths, temp.name) logging.info('CVO sorting took %s minutes', (time.time() - start_time) / 60) logging.info('Transforming call_variants_output to variants.') independent_variants = _transform_call_variants_output_to_variants( input_sorted_tfrecord_path=temp.name, qual_filter=FLAGS.qual_filter, multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter, sample_name=sample_name, group_variants=FLAGS.group_variants, use_multiallelic_model=FLAGS.use_multiallelic_model) variant_generator = haplotypes.maybe_resolve_conflicting_variants( independent_variants) header = dv_vcf_constants.deepvariant_header( contigs=contigs, sample_names=[sample_name]) use_csi = _decide_to_use_csi(contigs) start_time = time.time() if not FLAGS.nonvariant_site_tfrecord_path: logging.info('Writing variants to VCF.') write_variants_to_vcf( variant_iterable=variant_generator, output_vcf_path=FLAGS.outfile, header=header) if FLAGS.outfile.endswith('.gz'): build_index(FLAGS.outfile, use_csi) logging.info('VCF creation took %s minutes', (time.time() - start_time) / 60) else: logging.info('Merging and writing variants to VCF and gVCF.') lessthanfn = _get_contig_based_lessthan(contigs) with vcf.VcfWriter( FLAGS.outfile, header=header, round_qualities=True) as vcf_writer, \ vcf.VcfWriter( FLAGS.gvcf_outfile, header=header, round_qualities=True) \ as gvcf_writer: nonvariant_generator = tfrecord.read_shard_sorted_tfrecords( FLAGS.nonvariant_site_tfrecord_path, key=_get_contig_based_variant_sort_keyfn(contigs), proto=variants_pb2.Variant) merge_and_write_variants_and_nonvariants(variant_generator, nonvariant_generator, lessthanfn, fasta_reader, vcf_writer, gvcf_writer) if FLAGS.outfile.endswith('.gz'): build_index(FLAGS.outfile, use_csi) if FLAGS.gvcf_outfile.endswith('.gz'): build_index(FLAGS.gvcf_outfile, use_csi) logging.info('Finished writing VCF and gVCF in %s minutes.', (time.time() - start_time) / 60) if FLAGS.vcf_stats_report: outfile_base = _get_base_path(FLAGS.outfile) with vcf.VcfReader(FLAGS.outfile) as reader: vcf_stats.create_vcf_report( variants=reader.iterate(), output_basename=outfile_base, sample_name=sample_name, vcf_reader=reader) if record: temp.close()
def main(argv=()): with errors.clean_commandline_error_exit(): if len(argv) > 1: errors.log_and_raise( 'Command line parsing failure: postprocess_variants does not accept ' 'positional arguments but some are present on the command line: ' '"{}".'.format(str(argv)), errors.CommandLineError) del argv # Unused. if (not FLAGS.nonvariant_site_tfrecord_path) != (not FLAGS.gvcf_outfile): errors.log_and_raise( 'gVCF creation requires both nonvariant_site_tfrecord_path and ' 'gvcf_outfile flags to be set.', errors.CommandLineError) proto_utils.uses_fast_cpp_protos_or_die() logging_level.set_from_flag() fasta_reader = fasta.RefFastaReader(FLAGS.ref, cache_size=_FASTA_CACHE_SIZE) contigs = fasta_reader.header.contigs paths = io_utils.maybe_generate_sharded_filenames(FLAGS.infile) # Read one CallVariantsOutput record and extract the sample name from it. # Note that this assumes that all CallVariantsOutput protos in the infile # contain a single VariantCall within their constituent Variant proto, and # that the call_set_name is identical in each of the records. record = next( io_utils.read_tfrecords( paths[0], proto=deepvariant_pb2.CallVariantsOutput, max_records=1)) sample_name = _extract_single_sample_name(record) header = dv_vcf_constants.deepvariant_header( contigs=contigs, sample_names=[sample_name]) with tempfile.NamedTemporaryFile() as temp: postprocess_variants_lib.process_single_sites_tfrecords( contigs, paths, temp.name) independent_variants = _transform_call_variants_output_to_variants( input_sorted_tfrecord_path=temp.name, qual_filter=FLAGS.qual_filter, multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter, sample_name=sample_name) variant_generator = haplotypes.maybe_resolve_conflicting_variants( independent_variants) write_variants_to_vcf( variant_generator=variant_generator, output_vcf_path=FLAGS.outfile, header=header) # Also write out the gVCF file if it was provided. if FLAGS.nonvariant_site_tfrecord_path: nonvariant_generator = io_utils.read_shard_sorted_tfrecords( FLAGS.nonvariant_site_tfrecord_path, key=_get_contig_based_variant_sort_keyfn(contigs), proto=variants_pb2.Variant) with vcf.VcfReader(FLAGS.outfile, use_index=False) as variant_reader: lessthanfn = _get_contig_based_lessthan(contigs) gvcf_variants = ( _transform_to_gvcf_record(variant) for variant in variant_reader.iterate()) merged_variants = merge_variants_and_nonvariants( gvcf_variants, nonvariant_generator, lessthanfn, fasta_reader) write_variants_to_vcf( variant_generator=merged_variants, output_vcf_path=FLAGS.gvcf_outfile, header=header)
def run(): """Create pileup images from examples, filtered in various ways.""" with errors.clean_commandline_error_exit(): if FLAGS.column_labels: column_labels = FLAGS.column_labels.split(',') else: column_labels = None filter_to_vcf = FLAGS.vcf is not None if filter_to_vcf: ids_from_vcf = parse_vcf(FLAGS.vcf) logging.info( 'Found %d loci in VCF. ' 'Only examples matching these loci will be output.', len(ids_from_vcf)) filter_to_region = FLAGS.regions is not None if filter_to_region: passes_region_filter = create_region_filter( region_flag_string=FLAGS.regions, verbose=FLAGS.verbose) # Use nucleus.io.tfrecord to read all shards. dataset = tfrecord.read_tfrecords(FLAGS.examples) # Check flag here to avoid expensive string matching on every iteration. make_rgb = FLAGS.image_type in ['both', 'RGB'] make_channels = FLAGS.image_type in ['both', 'channels'] num_scanned = 0 num_output = 0 for example in dataset: num_scanned += 1 # Only when scanning many examples, print a dot for each one to # indicate that the script is making progress and not stalled. if num_scanned % UPDATE_EVERY_N_EXAMPLES == 0: if num_scanned == UPDATE_EVERY_N_EXAMPLES: print('Reporting progress below. Writing one dot every time {} ' 'examples have been scanned:'.format(UPDATE_EVERY_N_EXAMPLES)) # Print another dot on the same line, using print since logging does # not support printing without a newline. print('.', end='', flush=True) # Extract variant from example. variant = vis.variant_from_example(example) locus_id = vis.locus_id_from_variant(variant) indices = vis.alt_allele_indices_from_example(example) # Optionally filter to variants in the VCF. if filter_to_vcf: # Check if the locus is in the VCF. if locus_id not in ids_from_vcf: # Skip this example since it doesn't match the VCF. continue if filter_to_region and not passes_region_filter(variant): continue # Use locus ID in the filename, replacing long alleles with INS/DEL sizes. locus_with_alt_id = get_short_id(variant, indices) # Examples of long alleles replaced with their sizes: # 20:62456134_INS103bp.png # 20:62481177_DEL51bp.png # Examples of short alleles where the full string is included: # 1:55424995_TC->T.png # 1:55424996_CT->CTT.png # 1:55424996_CT->C.png # 1:55424996_CT->TTT.png # 1:55424996_CT->C|CTT.png if FLAGS.verbose: logging.info('\nOutputting image for: %s', locus_with_alt_id) full_id = get_full_id(variant, indices) if locus_with_alt_id != full_id: logging.info( 'ID above was shortened due to long ref/alt strings. ' 'Original: %s', full_id) # If the example has a truth label, optionally include it. optional_truth_label = '' if FLAGS.truth_labels: truth_label = get_label(example) if truth_label is not None: optional_truth_label = '_label{}'.format(truth_label) # Extract and format example into channels. channels = vis.channels_from_example(example) if column_labels is not None and len(column_labels) != len(channels): raise ValueError( '--column_labels must have {} names separated by commas, since ' 'there are {} channels in the examples. ' 'However, {} column labels were found: {}'.format( len(channels), len(channels), len(column_labels), ','.join(['"{}"'.format(x) for x in column_labels]))) output_prefix = '{}_'.format( FLAGS.output) if FLAGS.output is not None else '' # Create image with a grey-scale row of channels and save to file. if make_channels: channels_output = '{}channels_{}{}.png'.format(output_prefix, locus_with_alt_id, optional_truth_label) vis.draw_deepvariant_pileup( channels=channels, path=channels_output, scale=1, show=False, annotated=FLAGS.annotation, labels=column_labels) # Create RGB image and save to file. if make_rgb: rgb_output = '{}rgb_{}{}.png'.format(output_prefix, locus_with_alt_id, optional_truth_label) vis.draw_deepvariant_pileup( channels=channels, composite_type='RGB', path=rgb_output, scale=1, show=False, annotated=FLAGS.annotation, labels=column_labels) # Check if --num_records quota has been hit yet. num_output += 1 if FLAGS.num_records != -1 and num_output >= FLAGS.num_records: break logging.info('Scanned %d examples and output %d images.', num_scanned, num_output) if num_scanned == 0 and FLAGS.examples.startswith('gs://'): if sharded_file_utils.is_sharded_file_spec(FLAGS.examples): paths = sharded_file_utils.generate_sharded_filenames(FLAGS.examples) special_gcs_message = ('WARNING: --examples sharded files are either ' 'all empty or do not exist. Please check that ' 'the paths are correct:\n') for p in paths[0:3]: special_gcs_message += 'gsutil ls {}\n'.format(p) logging.warning(special_gcs_message) else: logging.warning( 'WARNING: --examples file is either empty or does not exist. ' 'Please check that the path is correct: \n' 'gsutil ls %s', FLAGS.examples)
def test_clean_commandline_error_exit_clean_exit(self, exc_type, exit_value): with mock.patch.object(sys, 'exit') as mock_exit: with errors.clean_commandline_error_exit(exit_value=exit_value): raise exc_type() mock_exit.assert_called_once_with(exit_value)
def test_clean_commandline_error_exit_raise_non_allowed(self, exc_type, msg): with self.assertRaisesRegexp(exc_type, msg): with errors.clean_commandline_error_exit(): raise exc_type(msg)