def setUp(self, mock_find_non_empty):
# We never query this table for running the following test, however, the
# mock values are based on this table's schema. In other words:
# mock_columns.return_value = self._flatter._get_column_names()
# mock_sub_fields.return_value = self._flatter._get_call_sub_fields()
input_base_table = ('gcp-variant-transforms-test:'
'bq_to_vcf_integration_tests.'
'merge_option_move_to_calls')
self._flatter = partitioning.FlattenCallColumn(input_base_table,
['chr20'], False)
# Set in mock_find_non_empty, ie mock of _find_one_non_empty_table()
mock_find_non_empty.return_value = None
self._flatter._schema_table_id = 'merge_option_move_to_calls__chr20'
def run(argv=None):
# type: (List[str]) -> None
"""Runs VCF to BigQuery pipeline."""
logging.info('Command: %s', ' '.join(argv or sys.argv))
known_args, pipeline_args = pipeline_common.parse_args(
argv, _COMMAND_LINE_OPTIONS)
if known_args.auto_flags_experiment:
_get_input_dimensions(known_args, pipeline_args)
annotated_vcf_pattern = _run_annotation_pipeline(known_args, pipeline_args)
all_patterns = ([annotated_vcf_pattern]
if annotated_vcf_pattern else known_args.all_patterns)
variant_merger = _get_variant_merge_strategy(known_args)
pipeline_mode = pipeline_common.get_pipeline_mode(all_patterns)
beam_pipeline_options = pipeline_options.PipelineOptions(pipeline_args)
avro_root_path = _get_avro_root_path(beam_pipeline_options)
# Starts a pipeline to merge VCF headers in beam if the total files that
# match the input pattern exceeds _SMALL_DATA_THRESHOLD
_merge_headers(known_args, pipeline_args, pipeline_mode, avro_root_path,
annotated_vcf_pattern)
# Retrieve merged headers prior to launching the pipeline. This is needed
# since the BigQuery schema cannot yet be dynamically created based on input.
# See https://issues.apache.org/jira/browse/BEAM-2801.
header_fields = vcf_header_parser.get_vcf_headers(
known_args.representative_header_file)
counter_factory = metrics_util.CounterFactory()
processed_variant_factory = processed_variant.ProcessedVariantFactory(
header_fields, known_args.split_alternate_allele_info_fields,
known_args.allow_malformed_records, known_args.annotation_fields,
known_args.use_allele_num, known_args.minimal_vep_alt_matching,
known_args.infer_annotation_types, counter_factory)
schema = schema_converter.generate_schema_from_header_fields(
header_fields, processed_variant_factory, variant_merger,
known_args.use_1_based_coordinate, known_args.include_call_name)
sharding = variant_sharding.VariantSharding(
known_args.sharding_config_path)
if sharding.should_keep_shard(sharding.get_residual_index()):
num_shards = sharding.get_num_shards()
else:
num_shards = sharding.get_num_shards() - 1
if known_args.update_schema_on_append:
for i in range(num_shards):
table_suffix = sharding.get_output_table_suffix(i)
table_name = bigquery_util.compose_table_name(
known_args.output_table, table_suffix)
bigquery_util.update_bigquery_schema_on_append(
schema.fields, table_name)
pipeline = beam.Pipeline(options=beam_pipeline_options)
variants = _read_variants(
all_patterns,
pipeline,
known_args,
pipeline_mode,
use_1_based_coordinate=known_args.use_1_based_coordinate)
if known_args.allow_malformed_records:
variants |= 'DropMalformedRecords' >> filter_variants.FilterVariants()
sharded_variants = variants | 'ShardVariants' >> beam.Partition(
shard_variants.ShardVariants(sharding), sharding.get_num_shards())
variants = []
for i in range(num_shards):
suffix = sharding.get_output_table_suffix(i)
# Convert tuples to list
variants.append(sharded_variants[i])
if variant_merger:
variants[i] |= ('MergeVariants' + suffix >>
merge_variants.MergeVariants(variant_merger))
variants[i] |= (
'ProcessVariants' + suffix >>
beam.Map(processed_variant_factory.create_processed_variant). \
with_output_types(processed_variant.ProcessedVariant))
_ = (variants[i]
| 'VariantToAvro' + suffix >> variant_to_avro.VariantToAvroFiles(
avro_root_path + suffix,
schema,
allow_incompatible_records=known_args.
allow_incompatible_records,
omit_empty_sample_calls=known_args.omit_empty_sample_calls,
null_numeric_value_replacement=(
known_args.null_numeric_value_replacement),
include_call_name=known_args.include_call_name))
result = pipeline.run()
try:
state = result.wait_until_finish()
if state != beam.runners.runner.PipelineState.DONE:
logging.error(
'Dataflow pipeline terminated in an unexpected state: %s',
state)
raise AssertionError(
'Dataflow pipeline terminated in {} state'.format(state))
except Exception as e:
logging.error('Dataflow pipeline failed.')
raise e
else:
logging.info('Dataflow pipeline finished successfully.')
metrics_util.log_all_counters(result)
# After pipeline is done, create output tables and load AVRO files into them.
schema_file = _write_schema_to_temp_file(schema, avro_root_path)
suffixes = []
try:
for i in range(num_shards):
suffixes.append(sharding.get_output_table_suffix(i))
partition_range_end = sharding.get_output_table_partition_range_end(
i)
if not known_args.append:
table_name = bigquery_util.compose_table_name(
known_args.output_table, suffixes[i])
partitioning.create_bq_table(
table_name, schema_file,
bigquery_util.ColumnKeyConstants.START_POSITION,
partition_range_end)
_record_newly_created_table(table_name)
logging.info('Integer range partitioned table %s was created.',
table_name)
if not known_args.append:
_record_newly_created_table(
sample_info_table_schema_generator.create_sample_info_table(
known_args.output_table))
suffixes.append(
sample_info_table_schema_generator.SAMPLE_INFO_TABLE_SUFFIX)
load_avro = avro_util.LoadAvro(avro_root_path, known_args.output_table,
suffixes, False)
not_empty_variant_suffixes = load_avro.start_loading()
logging.info('Following tables were loaded with at least 1 row:')
for suffix in not_empty_variant_suffixes:
logging.info(
bigquery_util.compose_table_name(known_args.output_table,
suffix))
# Remove sample_info table from both lists to avoid duplicating it when
# --sample_lookup_optimized_output_table flag is set
suffixes.remove(
sample_info_table_schema_generator.SAMPLE_INFO_TABLE_SUFFIX)
if sample_info_table_schema_generator.SAMPLE_INFO_TABLE_SUFFIX in\
not_empty_variant_suffixes:
not_empty_variant_suffixes.remove(
sample_info_table_schema_generator.SAMPLE_INFO_TABLE_SUFFIX)
except Exception as e:
logging.error(
'Something unexpected happened during the loading of AVRO '
'files to BigQuery: %s', str(e))
logging.info(
'Since the write to BigQuery stage failed, we did not delete '
'AVRO files in your GCS bucket. You can manually import them '
'to BigQuery. To avoid extra storage charges, delete them if '
'you do not need them, AVRO files are located at: %s',
avro_root_path)
raise e
else:
logging.warning('All AVRO files were successfully loaded to BigQuery.')
if known_args.keep_intermediate_avro_files:
logging.info(
'Since "--keep_intermediate_avro_files" flag is set, the '
'AVRO files are kept and stored at: %s', avro_root_path)
else:
if bigquery_util.delete_gcs_files(avro_root_path) != 0:
logging.error(
'Deletion of intermediate AVRO files located at "%s" has '
'failed.', avro_root_path)
if known_args.sample_lookup_optimized_output_table:
flatten_call_column = partitioning.FlattenCallColumn(
known_args.output_table, not_empty_variant_suffixes,
known_args.append)
try:
flatten_schema_file = tempfile.mkstemp(
suffix=_BQ_SCHEMA_FILE_SUFFIX)[1]
if not flatten_call_column.get_flatten_table_schema(
flatten_schema_file):
raise ValueError('Failed to extract schema of flatten table')
# Create output flatten tables if needed
if not known_args.append:
# Create all sample optimized tables including those that will be empty.
for suffix in suffixes:
output_table_id = bigquery_util.compose_table_name(
known_args.sample_lookup_optimized_output_table,
suffix)
partitioning.create_bq_table(
output_table_id, flatten_schema_file,
bigquery_util.ColumnKeyConstants.CALLS_SAMPLE_ID,
partitioning.MAX_RANGE_END)
_record_newly_created_table(output_table_id)
logging.info(
'Sample lookup optimized table %s was created.',
output_table_id)
# Copy to flatten sample lookup tables from the variant lookup tables.
# Note: uses WRITE_TRUNCATE to overwrite the existing tables (issue #607).
flatten_call_column.copy_to_flatten_table(
known_args.sample_lookup_optimized_output_table)
logging.info(
'All sample lookup optimized tables are fully loaded.')
except Exception as e:
logging.error(
'Something unexpected happened during the loading rows to '
'sample optimized table stage: %s', str(e))
raise e