def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: make_examples does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
hts_verbose.set(hts_verbose.htsLogLevel[FLAGS.hts_logging_level])
# Set up options; may do I/O.
options = default_options(add_flags=True, flags_obj=FLAGS)
# Check arguments that apply to any mode.
if not options.reference_filename:
errors.log_and_raise('ref argument is required.', errors.CommandLineError)
if not options.reads_filename:
errors.log_and_raise('reads argument is required.',
errors.CommandLineError)
if not options.examples_filename:
errors.log_and_raise('examples argument is required.',
errors.CommandLineError)
if options.n_cores != 1:
errors.log_and_raise(
'Currently only supports n_cores == 1 but got {}.'.format(
options.n_cores), errors.CommandLineError)
# Check for argument issues specific to train mode.
if in_training_mode(options):
if not options.truth_variants_filename:
errors.log_and_raise(
'truth_variants is required when in training mode.',
errors.CommandLineError)
if not options.confident_regions_filename:
errors.log_and_raise(
'confident_regions is required when in training mode.',
errors.CommandLineError)
if options.gvcf_filename:
errors.log_and_raise('gvcf is not allowed in training mode.',
errors.CommandLineError)
else:
# Check for argument issues specific to calling mode.
if options.variant_caller_options.sample_name == _UNKNOWN_SAMPLE:
errors.log_and_raise('sample_name must be specified in calling mode.',
errors.CommandLineError)
if options.variant_caller_options.gq_resolution < 1:
errors.log_and_raise('gq_resolution must be a non-negative integer.',
errors.CommandLineError)
# Run!
make_examples_runner(options)
def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: call_variants does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
model = modeling.get_model(FLAGS.model_name)
call_variants(
examples_filename=FLAGS.examples,
checkpoint_path=FLAGS.checkpoint,
model=model,
execution_hardware=FLAGS.execution_hardware,
output_file=FLAGS.outfile,
max_batches=FLAGS.max_batches,
batch_size=FLAGS.batch_size)
def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: call_variants does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
model = modeling.get_model(FLAGS.model_name)
call_variants(
examples_filename=FLAGS.examples,
checkpoint_path=FLAGS.checkpoint,
model=model,
execution_hardware=FLAGS.execution_hardware,
output_file=FLAGS.outfile,
max_batches=FLAGS.max_batches,
batch_size=FLAGS.batch_size)
def test_clean_commandline_error_exit_clean_exit(self, exc_type,
exit_value):
with mock.patch.object(sys, 'exit') as mock_exit:
with errors.clean_commandline_error_exit(exit_value=exit_value):
raise exc_type()
mock_exit.assert_called_once_with(exit_value)
def test_clean_commandline_error_exit_raise_non_allowed(
self, exc_type, msg):
with self.assertRaisesRegexp(exc_type, msg):
with errors.clean_commandline_error_exit():
raise exc_type(msg)
def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: postprocess_variants does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
if (not FLAGS.nonvariant_site_tfrecord_path) != (
not FLAGS.gvcf_outfile):
errors.log_and_raise(
'gVCF creation requires both nonvariant_site_tfrecord_path and '
'gvcf_outfile flags to be set.', errors.CommandLineError)
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
fasta_reader = fasta.IndexedFastaReader(FLAGS.ref,
cache_size=_FASTA_CACHE_SIZE)
contigs = fasta_reader.header.contigs
paths = io_utils.maybe_generate_sharded_filenames(FLAGS.infile)
# Read one CallVariantsOutput record and extract the sample name from it.
# Note that this assumes that all CallVariantsOutput protos in the infile
# contain a single VariantCall within their constituent Variant proto, and
# that the call_set_name is identical in each of the records.
record = tf_utils.get_one_example_from_examples_path(
','.join(paths), proto=deepvariant_pb2.CallVariantsOutput)
if record is None:
raise ValueError('Cannot find any records in {}'.format(
','.join(paths)))
sample_name = _extract_single_sample_name(record)
header = dv_vcf_constants.deepvariant_header(
contigs=contigs, sample_names=[sample_name])
with tempfile.NamedTemporaryFile() as temp:
postprocess_variants_lib.process_single_sites_tfrecords(
contigs, paths, temp.name)
independent_variants = _transform_call_variants_output_to_variants(
input_sorted_tfrecord_path=temp.name,
qual_filter=FLAGS.qual_filter,
multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter,
sample_name=sample_name)
variant_generator = haplotypes.maybe_resolve_conflicting_variants(
independent_variants)
write_variants_to_vcf(variant_generator=variant_generator,
output_vcf_path=FLAGS.outfile,
header=header)
# Also write out the gVCF file if it was provided.
if FLAGS.nonvariant_site_tfrecord_path:
nonvariant_generator = io_utils.read_shard_sorted_tfrecords(
FLAGS.nonvariant_site_tfrecord_path,
key=_get_contig_based_variant_sort_keyfn(contigs),
proto=variants_pb2.Variant)
with vcf.VcfReader(FLAGS.outfile) as variant_reader:
lessthanfn = _get_contig_based_lessthan(contigs)
gvcf_variants = (_transform_to_gvcf_record(variant)
for variant in variant_reader.iterate())
merged_variants = merge_variants_and_nonvariants(
gvcf_variants, nonvariant_generator, lessthanfn,
fasta_reader)
write_variants_to_vcf(variant_generator=merged_variants,
output_vcf_path=FLAGS.gvcf_outfile,
header=header)
def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: postprocess_variants does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
if (not FLAGS.nonvariant_site_tfrecord_path) != (not FLAGS.gvcf_outfile):
errors.log_and_raise(
'gVCF creation requires both nonvariant_site_tfrecord_path and '
'gvcf_outfile flags to be set.', errors.CommandLineError)
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
fasta_reader = fasta.IndexedFastaReader(
FLAGS.ref, cache_size=_FASTA_CACHE_SIZE)
contigs = fasta_reader.header.contigs
paths = sharded_file_utils.maybe_generate_sharded_filenames(FLAGS.infile)
# Read one CallVariantsOutput record and extract the sample name from it.
# Note that this assumes that all CallVariantsOutput protos in the infile
# contain a single VariantCall within their constituent Variant proto, and
# that the call_set_name is identical in each of the records.
record = tf_utils.get_one_example_from_examples_path(
','.join(paths), proto=deepvariant_pb2.CallVariantsOutput)
if record is None:
logging.info('call_variants_output is empty. Writing out empty VCF.')
sample_name = dv_constants.DEFAULT_SAMPLE_NAME
if FLAGS.sample_name:
logging.info(
'--sample_name is set in postprocess_variant. Using %s as the '
'sample name.', FLAGS.sample_name)
sample_name = FLAGS.sample_name
variant_generator = iter([])
else:
sample_name = _extract_single_sample_name(record)
temp = tempfile.NamedTemporaryFile()
start_time = time.time()
postprocess_variants_lib.process_single_sites_tfrecords(
contigs, paths, temp.name)
logging.info('CVO sorting took %s minutes',
(time.time() - start_time) / 60)
logging.info('Transforming call_variants_output to variants.')
independent_variants = _transform_call_variants_output_to_variants(
input_sorted_tfrecord_path=temp.name,
qual_filter=FLAGS.qual_filter,
multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter,
sample_name=sample_name,
group_variants=FLAGS.group_variants,
use_multiallelic_model=FLAGS.use_multiallelic_model)
variant_generator = haplotypes.maybe_resolve_conflicting_variants(
independent_variants)
header = dv_vcf_constants.deepvariant_header(
contigs=contigs, sample_names=[sample_name])
use_csi = _decide_to_use_csi(contigs)
start_time = time.time()
if not FLAGS.nonvariant_site_tfrecord_path:
logging.info('Writing variants to VCF.')
write_variants_to_vcf(
variant_iterable=variant_generator,
output_vcf_path=FLAGS.outfile,
header=header)
if FLAGS.outfile.endswith('.gz'):
build_index(FLAGS.outfile, use_csi)
logging.info('VCF creation took %s minutes',
(time.time() - start_time) / 60)
else:
logging.info('Merging and writing variants to VCF and gVCF.')
lessthanfn = _get_contig_based_lessthan(contigs)
with vcf.VcfWriter(
FLAGS.outfile, header=header, round_qualities=True) as vcf_writer, \
vcf.VcfWriter(
FLAGS.gvcf_outfile, header=header, round_qualities=True) \
as gvcf_writer:
nonvariant_generator = tfrecord.read_shard_sorted_tfrecords(
FLAGS.nonvariant_site_tfrecord_path,
key=_get_contig_based_variant_sort_keyfn(contigs),
proto=variants_pb2.Variant)
merge_and_write_variants_and_nonvariants(variant_generator,
nonvariant_generator,
lessthanfn, fasta_reader,
vcf_writer, gvcf_writer)
if FLAGS.outfile.endswith('.gz'):
build_index(FLAGS.outfile, use_csi)
if FLAGS.gvcf_outfile.endswith('.gz'):
build_index(FLAGS.gvcf_outfile, use_csi)
logging.info('Finished writing VCF and gVCF in %s minutes.',
(time.time() - start_time) / 60)
if FLAGS.vcf_stats_report:
outfile_base = _get_base_path(FLAGS.outfile)
with vcf.VcfReader(FLAGS.outfile) as reader:
vcf_stats.create_vcf_report(
variants=reader.iterate(),
output_basename=outfile_base,
sample_name=sample_name,
vcf_reader=reader)
if record:
temp.close()
def main(argv=()):
with errors.clean_commandline_error_exit():
if len(argv) > 1:
errors.log_and_raise(
'Command line parsing failure: postprocess_variants does not accept '
'positional arguments but some are present on the command line: '
'"{}".'.format(str(argv)), errors.CommandLineError)
del argv # Unused.
if (not FLAGS.nonvariant_site_tfrecord_path) != (not FLAGS.gvcf_outfile):
errors.log_and_raise(
'gVCF creation requires both nonvariant_site_tfrecord_path and '
'gvcf_outfile flags to be set.', errors.CommandLineError)
proto_utils.uses_fast_cpp_protos_or_die()
logging_level.set_from_flag()
fasta_reader = fasta.RefFastaReader(FLAGS.ref, cache_size=_FASTA_CACHE_SIZE)
contigs = fasta_reader.header.contigs
paths = io_utils.maybe_generate_sharded_filenames(FLAGS.infile)
# Read one CallVariantsOutput record and extract the sample name from it.
# Note that this assumes that all CallVariantsOutput protos in the infile
# contain a single VariantCall within their constituent Variant proto, and
# that the call_set_name is identical in each of the records.
record = next(
io_utils.read_tfrecords(
paths[0], proto=deepvariant_pb2.CallVariantsOutput, max_records=1))
sample_name = _extract_single_sample_name(record)
header = dv_vcf_constants.deepvariant_header(
contigs=contigs, sample_names=[sample_name])
with tempfile.NamedTemporaryFile() as temp:
postprocess_variants_lib.process_single_sites_tfrecords(
contigs, paths, temp.name)
independent_variants = _transform_call_variants_output_to_variants(
input_sorted_tfrecord_path=temp.name,
qual_filter=FLAGS.qual_filter,
multi_allelic_qual_filter=FLAGS.multi_allelic_qual_filter,
sample_name=sample_name)
variant_generator = haplotypes.maybe_resolve_conflicting_variants(
independent_variants)
write_variants_to_vcf(
variant_generator=variant_generator,
output_vcf_path=FLAGS.outfile,
header=header)
# Also write out the gVCF file if it was provided.
if FLAGS.nonvariant_site_tfrecord_path:
nonvariant_generator = io_utils.read_shard_sorted_tfrecords(
FLAGS.nonvariant_site_tfrecord_path,
key=_get_contig_based_variant_sort_keyfn(contigs),
proto=variants_pb2.Variant)
with vcf.VcfReader(FLAGS.outfile, use_index=False) as variant_reader:
lessthanfn = _get_contig_based_lessthan(contigs)
gvcf_variants = (
_transform_to_gvcf_record(variant)
for variant in variant_reader.iterate())
merged_variants = merge_variants_and_nonvariants(
gvcf_variants, nonvariant_generator, lessthanfn, fasta_reader)
write_variants_to_vcf(
variant_generator=merged_variants,
output_vcf_path=FLAGS.gvcf_outfile,
header=header)
def run():
"""Create pileup images from examples, filtered in various ways."""
with errors.clean_commandline_error_exit():
if FLAGS.column_labels:
column_labels = FLAGS.column_labels.split(',')
else:
column_labels = None
filter_to_vcf = FLAGS.vcf is not None
if filter_to_vcf:
ids_from_vcf = parse_vcf(FLAGS.vcf)
logging.info(
'Found %d loci in VCF. '
'Only examples matching these loci will be output.',
len(ids_from_vcf))
filter_to_region = FLAGS.regions is not None
if filter_to_region:
passes_region_filter = create_region_filter(
region_flag_string=FLAGS.regions, verbose=FLAGS.verbose)
# Use nucleus.io.tfrecord to read all shards.
dataset = tfrecord.read_tfrecords(FLAGS.examples)
# Check flag here to avoid expensive string matching on every iteration.
make_rgb = FLAGS.image_type in ['both', 'RGB']
make_channels = FLAGS.image_type in ['both', 'channels']
num_scanned = 0
num_output = 0
for example in dataset:
num_scanned += 1
# Only when scanning many examples, print a dot for each one to
# indicate that the script is making progress and not stalled.
if num_scanned % UPDATE_EVERY_N_EXAMPLES == 0:
if num_scanned == UPDATE_EVERY_N_EXAMPLES:
print('Reporting progress below. Writing one dot every time {} '
'examples have been scanned:'.format(UPDATE_EVERY_N_EXAMPLES))
# Print another dot on the same line, using print since logging does
# not support printing without a newline.
print('.', end='', flush=True)
# Extract variant from example.
variant = vis.variant_from_example(example)
locus_id = vis.locus_id_from_variant(variant)
indices = vis.alt_allele_indices_from_example(example)
# Optionally filter to variants in the VCF.
if filter_to_vcf:
# Check if the locus is in the VCF.
if locus_id not in ids_from_vcf:
# Skip this example since it doesn't match the VCF.
continue
if filter_to_region and not passes_region_filter(variant):
continue
# Use locus ID in the filename, replacing long alleles with INS/DEL sizes.
locus_with_alt_id = get_short_id(variant, indices)
# Examples of long alleles replaced with their sizes:
# 20:62456134_INS103bp.png
# 20:62481177_DEL51bp.png
# Examples of short alleles where the full string is included:
# 1:55424995_TC->T.png
# 1:55424996_CT->CTT.png
# 1:55424996_CT->C.png
# 1:55424996_CT->TTT.png
# 1:55424996_CT->C|CTT.png
if FLAGS.verbose:
logging.info('\nOutputting image for: %s', locus_with_alt_id)
full_id = get_full_id(variant, indices)
if locus_with_alt_id != full_id:
logging.info(
'ID above was shortened due to long ref/alt strings. '
'Original: %s', full_id)
# If the example has a truth label, optionally include it.
optional_truth_label = ''
if FLAGS.truth_labels:
truth_label = get_label(example)
if truth_label is not None:
optional_truth_label = '_label{}'.format(truth_label)
# Extract and format example into channels.
channels = vis.channels_from_example(example)
if column_labels is not None and len(column_labels) != len(channels):
raise ValueError(
'--column_labels must have {} names separated by commas, since '
'there are {} channels in the examples. '
'However, {} column labels were found: {}'.format(
len(channels), len(channels), len(column_labels),
','.join(['"{}"'.format(x) for x in column_labels])))
output_prefix = '{}_'.format(
FLAGS.output) if FLAGS.output is not None else ''
# Create image with a grey-scale row of channels and save to file.
if make_channels:
channels_output = '{}channels_{}{}.png'.format(output_prefix,
locus_with_alt_id,
optional_truth_label)
vis.draw_deepvariant_pileup(
channels=channels,
path=channels_output,
scale=1,
show=False,
annotated=FLAGS.annotation,
labels=column_labels)
# Create RGB image and save to file.
if make_rgb:
rgb_output = '{}rgb_{}{}.png'.format(output_prefix, locus_with_alt_id,
optional_truth_label)
vis.draw_deepvariant_pileup(
channels=channels,
composite_type='RGB',
path=rgb_output,
scale=1,
show=False,
annotated=FLAGS.annotation,
labels=column_labels)
# Check if --num_records quota has been hit yet.
num_output += 1
if FLAGS.num_records != -1 and num_output >= FLAGS.num_records:
break
logging.info('Scanned %d examples and output %d images.', num_scanned,
num_output)
if num_scanned == 0 and FLAGS.examples.startswith('gs://'):
if sharded_file_utils.is_sharded_file_spec(FLAGS.examples):
paths = sharded_file_utils.generate_sharded_filenames(FLAGS.examples)
special_gcs_message = ('WARNING: --examples sharded files are either '
'all empty or do not exist. Please check that '
'the paths are correct:\n')
for p in paths[0:3]:
special_gcs_message += 'gsutil ls {}\n'.format(p)
logging.warning(special_gcs_message)
else:
logging.warning(
'WARNING: --examples file is either empty or does not exist. '
'Please check that the path is correct: \n'
'gsutil ls %s', FLAGS.examples)
def test_clean_commandline_error_exit_clean_exit(self, exc_type, exit_value):
with mock.patch.object(sys, 'exit') as mock_exit:
with errors.clean_commandline_error_exit(exit_value=exit_value):
raise exc_type()
mock_exit.assert_called_once_with(exit_value)
def test_clean_commandline_error_exit_raise_non_allowed(self, exc_type, msg):
with self.assertRaisesRegexp(exc_type, msg):
with errors.clean_commandline_error_exit():
raise exc_type(msg)