def _validate_output_tables(self, client, output_table_base_name, sharding_config_path, append, is_main_output): if (output_table_base_name != bigquery_util.get_table_base_name(output_table_base_name)): raise ValueError( ('Output table cannot contain "{}". we reserve this ' 'string to mark sharded output tables.').format( bigquery_util.TABLE_SUFFIX_SEPARATOR)) project_id, dataset_id, table_id = bigquery_util.parse_table_reference( output_table_base_name) bigquery_util.raise_error_if_dataset_not_exists( client, project_id, dataset_id) all_output_tables = [] if is_main_output: all_output_tables.append( bigquery_util.compose_table_name(table_id, SAMPLE_INFO_TABLE_SUFFIX)) sharding = variant_sharding.VariantSharding(sharding_config_path) num_shards = sharding.get_num_shards() # In case there is no residual in config we will ignore the last shard. if not sharding.should_keep_shard(sharding.get_residual_index()): num_shards -= 1 for i in range(num_shards): table_suffix = sharding.get_output_table_suffix(i) if table_suffix != bigquery_util.get_table_base_name(table_suffix): raise ValueError( ('Table suffix cannot contain "{}" we reserve this ' 'string to mark sharded output tables.').format( bigquery_util.TABLE_SUFFIX_SEPARATOR)) all_output_tables.append( bigquery_util.compose_table_name(table_id, table_suffix)) for output_table in all_output_tables: if append: if not bigquery_util.table_exist(client, project_id, dataset_id, output_table): raise ValueError( 'Table {}:{}.{} does not exist, cannot append to it.'. format(project_id, dataset_id, output_table)) else: if bigquery_util.table_exist(client, project_id, dataset_id, output_table): raise ValueError(( 'Table {}:{}.{} already exists, cannot overwrite it. Please ' 'set `--append True` if you want to append to it.' ).format(project_id, dataset_id, output_table))
def test_table_exist(self): client = mock.Mock() client.tables.Get.return_value = bigquery.Table( tableReference=bigquery.TableReference( projectId='project', datasetId='dataset', tableId='table')) self.assertEqual( bigquery_util.table_exist(client, 'project', 'dataset', 'table'), True) client.tables.Get.side_effect = exceptions.HttpError( response={'status': '404'}, url='', content='') self.assertEqual( bigquery_util.table_exist(client, 'project', 'dataset', 'table'), False) client.tables.Get.side_effect = exceptions.HttpError( response={'status': '401'}, url='', content='') self.assertRaises(exceptions.HttpError, bigquery_util.table_exist, client, 'project', 'dataset', 'table')
def validate(self, parsed_args, client=None): if not client: credentials = GoogleCredentials.get_application_default().create_scoped( ['https://www.googleapis.com/auth/bigquery']) client = bigquery.BigqueryV2(credentials=credentials) project_id, dataset_id, table_id = bigquery_util.parse_table_reference( parsed_args.input_table) if not bigquery_util.table_exist(client, project_id, dataset_id, table_id): raise ValueError('Table {}:{}.{} does not exist.'.format( project_id, dataset_id, table_id)) if table_id.count(TABLE_SUFFIX_SEPARATOR) != 1: raise ValueError( 'Input table {} is malformed - exactly one suffix separator "{}" is ' 'required'.format(parsed_args.input_table, TABLE_SUFFIX_SEPARATOR)) base_table_id = table_id[:table_id.find(TABLE_SUFFIX_SEPARATOR)] sample_table_id = bigquery_util.compose_table_name(base_table_id, SAMPLE_INFO_TABLE_SUFFIX) if not bigquery_util.table_exist(client, project_id, dataset_id, sample_table_id): raise ValueError('Sample table {}:{}.{} does not exist.'.format( project_id, dataset_id, sample_table_id))