def test_to_schema_field(self): expected_schema_field1 = SchemaField(name='test', field_type='STRING', mode='NULLABLE', description='test_description') schema_field1_1 = BigQuerySchemaField('test', 'STRING', 'NULLABLE', 'test_description') actual_schema_field1_1 = BigQuerySchemaField.to_schema_field( schema_field1_1) self.assertEqual(expected_schema_field1, actual_schema_field1_1) schema_field1_2 = BigQuerySchemaField('test', 'INTEGER', 'REQUIRED', 'foo_bar') actual_schema_field1_2 = BigQuerySchemaField.to_schema_field( schema_field1_2) self.assertNotEqual(expected_schema_field1, actual_schema_field1_2) expected_schema_field2 = SchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(expected_schema_field1, )) schema_field2_1 = BigQuerySchemaField('test', 'RECORD', 'NULLABLE', 'test_description', (schema_field1_1, )) actual_schema_field2_1 = BigQuerySchemaField.to_schema_field( schema_field2_1) self.assertEqual(expected_schema_field2, actual_schema_field2_1) schema_field2_2 = BigQuerySchemaField('test', 'RECORD', 'NULLABLE', 'test_description', (schema_field1_2, )) actual_schema_field2_2 = BigQuerySchemaField.to_schema_field( schema_field2_2) self.assertNotEqual(expected_schema_field2, actual_schema_field2_2)
def build_query_field(source_schema, target_schema, prefix=None): query_fields = [] for target in target_schema: source = next((s for s in source_schema if s.name == target.name), None) if source: if target.field_type == 'RECORD': field_prefix = '{0}.{1}'.format( prefix, target.name) if prefix else target.name fields = TableAction.build_query_field( source.fields, target.fields, field_prefix) query_fields.append('struct({fields}) AS {alias}'.format( fields=fields, alias=target.name)) else: name = '{0}.{1}'.format( prefix, target.name) if prefix else target.name field_type = BigQuerySchemaField.normalize_field_type( target.field_type) query_fields.append( 'cast({name} AS {type}) AS {alias}'.format( name=name, type=field_type, alias=target.name)) else: if target.field_type == 'RECORD': field_prefix = '{0}.{1}'.format( prefix, target.name) if prefix else target.name fields = TableAction.build_query_field((), target.fields, field_prefix) query_fields.append('struct({fields}) AS {alias}'.format( fields=fields, alias=target.name)) else: query_fields.append( 'null AS {alias}'.format(alias=target.name)) return ', '.join(query_fields)
def from_table(table): schema = tuple( BigQuerySchemaField.from_schema_field(s) for s in table.schema) if table.schema else None return BigQueryTable( table_id=table.table_id, friendly_name=table.friendly_name, description=table.description, expires=table.expires, partitioning_type=table.partitioning_type, view_use_legacy_sql=table.view_use_legacy_sql if table.view_query else None, view_query=table.view_query if table.view_query else None, schema=None if table.view_query else schema, labels=table.labels)
def from_dict(value): schema = value.get('schema', None) if schema: schema = tuple(BigQuerySchemaField.from_dict(s) for s in schema) expires = value.get('expires', None) if expires: expires = parse_expires(expires) return BigQueryTable( table_id=value.get('table_id', None), friendly_name=value.get('friendly_name', None), description=value.get('description', None), expires=expires, partitioning_type=value.get('partitioning_type', None), view_use_legacy_sql=value.get('view_use_legacy_sql', None), view_query=value.get('view_query', None), schema=schema, labels=value.get('labels', None), )
def to_table(dataset_ref, model): schema = model.schema if schema: schema = tuple( BigQuerySchemaField.to_schema_field(s) for s in schema) else: schema = None table_ref = TableReference(dataset_ref, model.table_id) table = Table(table_ref, schema) table.friendly_name = model.friendly_name table.description = model.description table.expires = model.expires table.partitioning_type = model.partitioning_type if model.view_use_legacy_sql is not None: table.view_use_legacy_sql = model.view_use_legacy_sql if model.view_query is not None: table.view_query = model.view_query table.labels = model.labels if model.labels is not None else dict() return table
def test_from_dict(self): expected_schema_field1 = BigQuerySchemaField( name='test', field_type='STRING', mode='NULLABLE', description='test_description') actual_schema_field1_1 = BigQuerySchemaField.from_dict({ 'name': 'test', 'field_type': 'STRING', 'mode': 'NULLABLE', 'description': 'test_description' }) self.assertEqual(expected_schema_field1, actual_schema_field1_1) actual_schema_field1_2 = BigQuerySchemaField.from_dict({ 'name': 'test', 'field_type': 'INTEGER', 'mode': 'REQUIRED', 'description': 'foo_bar' }) self.assertNotEqual(expected_schema_field1, actual_schema_field1_2) expected_schema_field2 = BigQuerySchemaField( name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(expected_schema_field1, )) actual_schema_field2_1 = BigQuerySchemaField.from_dict({ 'name': 'test', 'field_type': 'RECORD', 'mode': 'NULLABLE', 'description': 'test_description', 'fields': [{ 'name': 'test', 'field_type': 'STRING', 'mode': 'NULLABLE', 'description': 'test_description' }] }) self.assertEqual(expected_schema_field2, actual_schema_field2_1) actual_schema_field2_2 = BigQuerySchemaField.from_dict({ 'name': 'test', 'field_type': 'RECORD', 'mode': 'NULLABLE', 'description': 'test_description', 'fields': [{ 'name': 'test', 'field_type': 'INTEGER', 'mode': 'REQUIRED', 'description': 'foo_bar' }] }) self.assertNotEqual(expected_schema_field2, actual_schema_field2_2)
def test_normalize_field_type(self): self.assertEqual('INT64', BigQuerySchemaField.normalize_field_type('INTEGER')) self.assertEqual('INT64', BigQuerySchemaField.normalize_field_type('INT64')) self.assertEqual('FLOAT64', BigQuerySchemaField.normalize_field_type('FLOAT')) self.assertEqual('FLOAT64', BigQuerySchemaField.normalize_field_type('FLOAT64')) self.assertEqual('BOOL', BigQuerySchemaField.normalize_field_type('BOOLEAN')) self.assertEqual('BOOL', BigQuerySchemaField.normalize_field_type('BOOL')) self.assertEqual('BYTES', BigQuerySchemaField.normalize_field_type('BYTES')) self.assertEqual('DATE', BigQuerySchemaField.normalize_field_type('DATE')) self.assertEqual('DATETIME', BigQuerySchemaField.normalize_field_type('DATETIME')) self.assertEqual('TIME', BigQuerySchemaField.normalize_field_type('TIME')) self.assertEqual('TIMESTAMP', BigQuerySchemaField.normalize_field_type('TIMESTAMP')) self.assertEqual('STRING', BigQuerySchemaField.normalize_field_type('STRING')) self.assertRaises( ValueError, lambda: BigQuerySchemaField.normalize_field_type('DECIMAL'))
def test_eq(self): schema_field1_1 = BigQuerySchemaField(name='test', field_type='STRING', mode='NULLABLE', description='test_description') schema_field1_2 = BigQuerySchemaField(name='test', field_type='STRING', mode='NULLABLE', description='test_description') self.assertEqual(schema_field1_1, schema_field1_2) schema_field2_1 = BigQuerySchemaField(name='test', field_type='INTEGER', mode='NULLABLE', description='test_description') schema_field2_2 = BigQuerySchemaField(name='test', field_type='STRING', mode='REQUIRED', description='foo_bar') self.assertNotEqual(schema_field2_1, schema_field2_2) schema_field3_1 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(schema_field1_1, )) schema_field3_2 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(schema_field1_1, )) schema_field3_3 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=[schema_field1_1]) self.assertEqual(schema_field3_1, schema_field3_2) self.assertEqual(schema_field3_1, schema_field3_3) schema_field4_1 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(schema_field1_1, )) schema_field4_2 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(schema_field1_1, schema_field2_1)) schema_field4_3 = BigQuerySchemaField(name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=(schema_field2_1, schema_field1_1)) schema_field4_4 = BigQuerySchemaField( name='test', field_type='RECORD', mode='NULLABLE', description='test_description', fields=[schema_field2_1, schema_field1_1]) self.assertNotEqual(schema_field4_1, schema_field4_2) self.assertNotEqual(schema_field4_1, schema_field4_3) self.assertNotEqual(schema_field4_1, schema_field4_4) self.assertEqual(schema_field4_2, schema_field4_3) self.assertEqual(schema_field4_2, schema_field4_4)
def test_to_table(self): project = 'test' dataset_ref = DatasetReference(project, 'test') expected_table1 = make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description') actual_table1_1 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description')) self.assertEqual(expected_table1.table_id, actual_table1_1.table_id) self.assertEqual(expected_table1.friendly_name, actual_table1_1.friendly_name) self.assertEqual(expected_table1.description, actual_table1_1.description) actual_table1_2 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='aaa', friendly_name='foo_bar', description='fizz_buzz')) self.assertNotEqual(expected_table1.table_id, actual_table1_2.table_id) self.assertNotEqual(expected_table1.friendly_name, actual_table1_2.friendly_name) self.assertNotEqual(expected_table1.description, actual_table1_2.description) expected_table2 = make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY') actual_table2_1 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY')) self.assertEqual(expected_table2.expires, actual_table2_1.expires) self.assertEqual(expected_table2.partitioning_type, actual_table2_1.partitioning_type) actual_table2_2 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='description', expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type=None)) self.assertNotEqual(expected_table2.expires, actual_table2_2.expires) self.assertNotEqual(expected_table2.partitioning_type, actual_table2_2.partitioning_type) expected_table3 = make_table( project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test') actual_table3_1 = BigQueryTable.to_table( dataset_ref, BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test')) self.assertEqual(expected_table3.view_use_legacy_sql, actual_table3_1.view_use_legacy_sql) self.assertEqual(expected_table3.view_query, actual_table3_1.view_query) actual_table3_2 = BigQueryTable.to_table( dataset_ref, BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=True, view_query='SELECT * FROM bigquery_datasetmanager.foo.bar')) self.assertNotEqual(expected_table3.view_use_legacy_sql, actual_table3_2.view_use_legacy_sql) self.assertNotEqual(expected_table3.view_query, actual_table3_2.view_query) expected_table4 = make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(SchemaField( name='test', field_type='INTEGER', mode='NULLABLE', description='test_description'), )) actual_table4_1 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(BigQuerySchemaField('test', 'INTEGER', 'NULLABLE', 'test_description', None), ))) self.assertEqual(expected_table4.schema, actual_table4_1.schema) actual_table4_2 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(SchemaField(name='test', field_type='STRING', mode='REQUIRED', description='foo_bar'), ))) self.assertNotEqual(expected_table4.schema, actual_table4_2.schema) expected_table5 = make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', labels={'foo': 'bar'}) actual_table5_1 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels={'foo': 'bar'})) self.assertEqual(expected_table5.labels, actual_table5_1.labels) actual_table5_2 = BigQueryTable.to_table( dataset_ref, BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels={ 'foo': 'bar', 'fizz': 'buzz' })) self.assertNotEqual(expected_table5.labels, actual_table5_2.labels)
def test_from_table(self): project = 'test' expected_table1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') actual_table1_1 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description')) self.assertEqual(expected_table1, actual_table1_1) actual_table1_2 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='foo_bar', description='fizz_buzz')) self.assertNotEqual(expected_table1, actual_table1_2) expected_table2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY') actual_table2_1 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY')) self.assertEqual(expected_table2, actual_table2_1) actual_table2_2 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type=None)) self.assertNotEqual(expected_table2, actual_table2_2) expected_table3 = BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test') actual_table3_1 = BigQueryTable.from_table( make_table( project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test')) self.assertEqual(expected_table3, actual_table3_1) actual_table3_2 = BigQueryTable.from_table( make_table( project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=True, view_query='SELECT * FROM bigquery_datasetmanager.foo.bar')) self.assertNotEqual(expected_table3, actual_table3_2) expected_table4 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(BigQuerySchemaField( name='test', field_type='INTEGER', mode='NULLABLE', description='test_description'), ), labels=None) actual_table4_1 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(SchemaField(name='test', field_type='INTEGER', mode='NULLABLE', description='test_description'), ))) self.assertEqual(expected_table4, actual_table4_1) actual_table4_2 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(SchemaField(name='test', field_type='STRING', mode='REQUIRED', description='foo_bar'), ))) self.assertNotEqual(expected_table4, actual_table4_2) expected_table5 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels={'foo': 'bar'}) actual_table5_1 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', labels={'foo': 'bar'})) self.assertEqual(expected_table5, actual_table5_1) actual_table5_2 = BigQueryTable.from_table( make_table(project=project, dataset_id='test', table_id='test', friendly_name='test_friendly_name', description='test_description', labels={ 'foo': 'bar', 'fizz': 'buzz' })) self.assertNotEqual(expected_table5, actual_table5_2)
def test_eq(self): table1_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') table1_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') self.assertEqual(table1_1, table1_2) table2_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') table2_2 = BigQueryTable( table_id='test', friendly_name='fizz_buzz', description='foo_bar', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY', ) self.assertNotEqual(table2_1, table2_2) table3_1 = BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', ) table3_2 = BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test') self.assertNotEqual(table3_1, table3_2) schema_field1 = BigQuerySchemaField(name='test', field_type='INTEGER', mode='NULLABLE', description='test_description') schema_field2 = BigQuerySchemaField(name='test', field_type='STRING', mode='REQUIRED', description='foo_bar') table4_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, )) table4_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, )) self.assertEqual(table4_1, table4_2) table5_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, )) table5_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field2, )) self.assertNotEqual(table5_1, table5_2) table6_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, schema_field2)) table6_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field2, schema_field1)) table6_3 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=[schema_field1, schema_field2]) self.assertEqual(table6_1, table6_2) self.assertEqual(table6_1, table6_3) self.assertEqual(table6_2, table6_3) table7_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') table7_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, )) table7_3 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, schema_field2)) self.assertNotEqual(table7_1, table7_2) self.assertNotEqual(table7_1, table7_3) self.assertNotEqual(table7_2, table7_3) label1 = {'foo': 'bar'} label2 = {'fizz': 'buzz'} table8_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label1) table8_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label1) self.assertEqual(table8_1, table8_2) table9_1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label1) table9_2 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label2) self.assertNotEqual(table9_1, table9_2)
def test_from_dict(self): expected_table1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') actual_table1_1 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description' }) self.assertEqual(expected_table1, actual_table1_1) actual_table1_2 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'fizz_buzz', 'description': 'foo_bar', 'expires': '2018-01-01T00:00:00.000000+00:00', 'partitioning_type': 'DAY' }) self.assertNotEqual(expected_table1, actual_table1_2) expected_table2 = BigQueryTable( table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test') actual_table2_1 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'view_use_legacy_sql': False, 'view_query': 'SELECT * FROM bigquery_datasetmanager.test.test' }) self.assertEqual(expected_table2, actual_table2_1) actual_table2_2 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'view_use_legacy_sql': True, 'view_query': 'SELECT * FROM bigquery_datasetmanager.foo.bar' }) self.assertNotEqual(expected_table2, actual_table2_2) expected_table3 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(BigQuerySchemaField( 'test', 'INTEGER', 'NULLABLE', 'test_description', None), )) actual_table3_1 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'schema': [{ 'name': 'test', 'field_type': 'INTEGER', 'mode': 'NULLABLE', 'description': 'test_description', 'fields': None }] }) self.assertEqual(expected_table3, actual_table3_1) actual_table3_2 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'schema': [{ 'name': 'test', 'field_type': 'STRING', 'mode': 'REQUIRED', 'description': 'foo_bar', 'fields': None }] }) self.assertNotEqual(expected_table3, actual_table3_2) expected_table4 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels={'foo': 'bar'}) actual_table4_1 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'labels': { 'foo': 'bar' } }) self.assertEqual(expected_table4, actual_table4_1) actual_table4_2 = BigQueryTable.from_dict({ 'table_id': 'test', 'friendly_name': 'test_friendly_name', 'description': 'test_description', 'labels': { 'foo': 'bar', 'fizz': 'buzz' } }) self.assertNotEqual(expected_table4, actual_table4_2)
def test_dump_table(self): table1 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description') expected_dump_data1 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: null labels: null """ actual_dump_data1 = dump(table1) self.assertEqual(expected_dump_data1, actual_dump_data1) table2 = BigQueryTable( table_id='test', friendly_name='fizz_buzz', description='foo_bar', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY', ) expected_dump_data2 = """table_id: test friendly_name: fizz_buzz description: foo_bar expires: 2018-01-01T00:00:00.000000+0000 partitioning_type: DAY view_use_legacy_sql: null view_query: null schema: null labels: null """ actual_dump_data2 = dump(table2) self.assertEqual(expected_dump_data2, actual_dump_data2) table3 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query="""SELECT * FROM bigquery_datasetmanager.test.test""") expected_dump_data3 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: false view_query: |- SELECT * FROM bigquery_datasetmanager.test.test schema: null labels: null """ actual_dump_data3 = dump(table3) self.assertEqual(expected_dump_data3, actual_dump_data3) schema_field1 = BigQuerySchemaField(name='test1', field_type='INTEGER', mode='NULLABLE', description='test_description') schema_field2 = BigQuerySchemaField(name='test2', field_type='STRING', mode='REQUIRED', description='foo_bar') schema_field3 = BigQuerySchemaField(name='test3', field_type='RECORD', mode='NULLABLE', description='fizz_buzz', fields=(schema_field1, schema_field2)) schema_field4 = BigQuerySchemaField(name='test4', field_type='RECORD', mode='NULLABLE', description='aaa_bbb', fields=(schema_field3, )) table4 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, )) expected_dump_data4 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: - name: test1 field_type: INTEGER mode: NULLABLE description: test_description fields: null labels: null """ actual_dump_data4 = dump(table4) self.assertEqual(expected_dump_data4, actual_dump_data4) table5 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, schema_field2)) expected_dump_data5 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: - name: test1 field_type: INTEGER mode: NULLABLE description: test_description fields: null - name: test2 field_type: STRING mode: REQUIRED description: foo_bar fields: null labels: null """ actual_dump_data5 = dump(table5) self.assertEqual(expected_dump_data5, actual_dump_data5) table6 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field3, )) expected_dump_data6 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: - name: test3 field_type: RECORD mode: NULLABLE description: fizz_buzz fields: - name: test1 field_type: INTEGER mode: NULLABLE description: test_description fields: null - name: test2 field_type: STRING mode: REQUIRED description: foo_bar fields: null labels: null """ actual_dump_data6 = dump(table6) self.assertEqual(expected_dump_data6, actual_dump_data6) table7 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', schema=(schema_field4, )) expected_dump_data7 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: - name: test4 field_type: RECORD mode: NULLABLE description: aaa_bbb fields: - name: test3 field_type: RECORD mode: NULLABLE description: fizz_buzz fields: - name: test1 field_type: INTEGER mode: NULLABLE description: test_description fields: null - name: test2 field_type: STRING mode: REQUIRED description: foo_bar fields: null labels: null """ actual_dump_data7 = dump(table7) self.assertEqual(expected_dump_data7, actual_dump_data7) label1 = {'foo': 'bar'} label2 = {'fizz': 'buzz', 'aaa': 'bbb'} table8 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label1) expected_dump_data8 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: null labels: foo: bar """ actual_dump_data8 = dump(table8) self.assertEqual(expected_dump_data8, actual_dump_data8) table9 = BigQueryTable(table_id='test', friendly_name='test_friendly_name', description='test_description', labels=label2) expected_dump_data9 = """table_id: test friendly_name: test_friendly_name description: test_description expires: null partitioning_type: null view_use_legacy_sql: null view_query: null schema: null labels: aaa: bbb fizz: buzz """ actual_dump_data9 = dump(table9) self.assertEqual(expected_dump_data9, actual_dump_data9)
def test_get_change_tables(self): schema_field1 = BigQuerySchemaField( name='test', field_type='INTEGER', mode='NULLABLE', description='test_description' ) schema_field2 = BigQuerySchemaField( name='test', field_type='STRING', mode='REQUIRED', description='foo_bar' ) label1 = { 'foo': 'bar' } label2 = { 'fizz': 'buzz' } source_table1_1 = BigQueryTable( table_id='test1', friendly_name='test_friendly_name', description='test_description' ) source_table1_2 = BigQueryTable( table_id='test2', friendly_name='test_friendly_name', description='test_description', expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC), partitioning_type='DAY' ) source_table1_3 = BigQueryTable( table_id='test3', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=False, view_query='SELECT * FROM bigquery_datasetmanager.test.test' ) source_table1_4 = BigQueryTable( table_id='test4', friendly_name='test_friendly_name', description='test_description', schema=(schema_field1, ) ) source_table1_5 = BigQueryTable( table_id='test5', friendly_name='test_friendly_name', description='test_description', labels=label1 ) target_table1_1 = BigQueryTable( table_id='test1', friendly_name='foo_bar', description='fizz_buzz' ) target_table1_2 = BigQueryTable( table_id='test2', friendly_name='test_friendly_name', description='test_description', expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC) ) target_table1_3 = BigQueryTable( table_id='test3', friendly_name='test_friendly_name', description='test_description', view_use_legacy_sql=True, view_query='SELECT * FROM bigquery_datasetmanager.foo.bar' ) target_table1_4 = BigQueryTable( table_id='test4', friendly_name='test_friendly_name', description='test_description', schema=(schema_field2, ) ) target_table1_5 = BigQueryTable( table_id='test5', friendly_name='test_friendly_name', description='test_description', labels=label2 ) source1 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] target1 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] actual_count1, actual_results1 = TableAction.get_change_tables(source1, target1) self.assertEqual(actual_count1, 5) self.assertEqual(set(actual_results1), {target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5}) source2 = [source_table1_5, source_table1_4, source_table1_3, source_table1_2, source_table1_1] target2 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] actual_count2, actual_results2 = TableAction.get_change_tables(source2, target2) self.assertEqual(actual_count2, 5) self.assertEqual(set(actual_results2), {target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5}) source3 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] target3 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] actual_count3, actual_results3 = TableAction.get_change_tables(source3, target3) self.assertEqual(actual_count3, 0) self.assertEqual(actual_results3, ()) source4 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] target4 = [source_table1_5, source_table1_4, source_table1_3, source_table1_2, source_table1_1] actual_count4, actual_results4 = TableAction.get_change_tables(source4, target4) self.assertEqual(actual_count4, 0) self.assertEqual(actual_results4, ()) source5 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] target5 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] actual_count5, actual_results5 = TableAction.get_change_tables(source5, target5) self.assertEqual(actual_count5, 0) self.assertEqual(actual_results5, ()) source6 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] target6 = [target_table1_5, target_table1_4, target_table1_3, target_table1_2, target_table1_1] actual_count6, actual_results6 = TableAction.get_change_tables(source6, target6) self.assertEqual(actual_count6, 0) self.assertEqual(actual_results6, ()) source7 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] target7 = [target_table1_1, target_table1_2] actual_count7, actual_results7 = TableAction.get_change_tables(source7, target7) self.assertEqual(actual_count7, 2) self.assertEqual(set(actual_results7), {target_table1_1, target_table1_2}) source8 = [source_table1_1, source_table1_2, source_table1_3, source_table1_4, source_table1_5] target8 = [target_table1_3, target_table1_4, target_table1_5] actual_count8, actual_results8 = TableAction.get_change_tables(source8, target8) self.assertEqual(actual_count8, 3) self.assertEqual(set(actual_results8), {target_table1_3, target_table1_4, target_table1_5}) source9 = [source_table1_1, source_table1_2] target9 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] actual_count9, actual_results9 = TableAction.get_change_tables(source9, target9) self.assertEqual(actual_count9, 2) self.assertEqual(set(actual_results9), {target_table1_1, target_table1_2}) source10 = [source_table1_3, source_table1_4, source_table1_5] target10 = [target_table1_1, target_table1_2, target_table1_3, target_table1_4, target_table1_5] actual_count10, actual_results10 = TableAction.get_change_tables(source10, target10) self.assertEqual(actual_count10, 3) self.assertEqual(set(actual_results10), {target_table1_3, target_table1_4, target_table1_5})
def test_build_query_field(self): source_schema_field1 = BigQuerySchemaField( name='test1', field_type='STRING', mode='NULLABLE', description='test_description' ) source_schema_field2 = BigQuerySchemaField( name='test2', field_type='INTEGER', mode='NULLABLE', description='test_description' ) source_schema_field3 = BigQuerySchemaField( name='test3', field_type='RECORD', mode='NULLABLE', description='test_description', fields=( BigQuerySchemaField( name='foo_bar', field_type='STRING', mode='NULLABLE', description='test_description' ), ) ) source_schema_field4 = BigQuerySchemaField( name='test4', field_type='RECORD', mode='NULLABLE', description='test_description', fields=( BigQuerySchemaField( name='fizz', field_type='INTEGER', mode='NULLABLE', description='test_description' ), BigQuerySchemaField( name='buzz', field_type='BOOL', mode='NULLABLE', description='test_description' ), ) ) target_schema_field1 = BigQuerySchemaField( name='test1', field_type='INTEGER', mode='NULLABLE', description='test_description' ) target_schema_field2 = BigQuerySchemaField( name='test2', field_type='STRING', mode='NULLABLE', description='test_description' ) target_schema_field3 = BigQuerySchemaField( name='test3', field_type='RECORD', mode='NULLABLE', description='test_description', fields=( BigQuerySchemaField( name='foo_bar', field_type='INTEGER', mode='NULLABLE', description='test_description' ), ) ) target_schema_field4 = BigQuerySchemaField( name='test4', field_type='RECORD', mode='NULLABLE', description='test_description', fields=( BigQuerySchemaField( name='fizz', field_type='FLOAT', mode='NULLABLE', description='test_description' ), BigQuerySchemaField( name='buzz', field_type='STRING', mode='NULLABLE', description='test_description' ), ) ) source1 = (source_schema_field1, source_schema_field2, source_schema_field3, source_schema_field4) target1 = (target_schema_field1, target_schema_field2, target_schema_field3, target_schema_field4) expected_query_field = 'cast(test1 AS INT64) AS test1, ' \ 'cast(test2 AS STRING) AS test2, ' \ 'struct(cast(test3.foo_bar AS INT64) AS foo_bar) AS test3, ' \ 'struct(cast(test4.fizz AS FLOAT64) AS fizz, ' \ 'cast(test4.buzz AS STRING) AS buzz) AS test4' actual_query_field = TableAction.build_query_field(source1, target1) self.assertEqual(expected_query_field, actual_query_field)