def test_to_schema_field(self):
expected_schema_field1 = SchemaField(name='test',
field_type='STRING',
mode='NULLABLE',
description='test_description')
schema_field1_1 = BigQuerySchemaField('test', 'STRING', 'NULLABLE',
'test_description')
actual_schema_field1_1 = BigQuerySchemaField.to_schema_field(
schema_field1_1)
self.assertEqual(expected_schema_field1, actual_schema_field1_1)
schema_field1_2 = BigQuerySchemaField('test', 'INTEGER', 'REQUIRED',
'foo_bar')
actual_schema_field1_2 = BigQuerySchemaField.to_schema_field(
schema_field1_2)
self.assertNotEqual(expected_schema_field1, actual_schema_field1_2)
expected_schema_field2 = SchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(expected_schema_field1, ))
schema_field2_1 = BigQuerySchemaField('test', 'RECORD', 'NULLABLE',
'test_description',
(schema_field1_1, ))
actual_schema_field2_1 = BigQuerySchemaField.to_schema_field(
schema_field2_1)
self.assertEqual(expected_schema_field2, actual_schema_field2_1)
schema_field2_2 = BigQuerySchemaField('test', 'RECORD', 'NULLABLE',
'test_description',
(schema_field1_2, ))
actual_schema_field2_2 = BigQuerySchemaField.to_schema_field(
schema_field2_2)
self.assertNotEqual(expected_schema_field2, actual_schema_field2_2)
def build_query_field(source_schema, target_schema, prefix=None):
query_fields = []
for target in target_schema:
source = next((s for s in source_schema if s.name == target.name),
None)
if source:
if target.field_type == 'RECORD':
field_prefix = '{0}.{1}'.format(
prefix, target.name) if prefix else target.name
fields = TableAction.build_query_field(
source.fields, target.fields, field_prefix)
query_fields.append('struct({fields}) AS {alias}'.format(
fields=fields, alias=target.name))
else:
name = '{0}.{1}'.format(
prefix, target.name) if prefix else target.name
field_type = BigQuerySchemaField.normalize_field_type(
target.field_type)
query_fields.append(
'cast({name} AS {type}) AS {alias}'.format(
name=name, type=field_type, alias=target.name))
else:
if target.field_type == 'RECORD':
field_prefix = '{0}.{1}'.format(
prefix, target.name) if prefix else target.name
fields = TableAction.build_query_field((), target.fields,
field_prefix)
query_fields.append('struct({fields}) AS {alias}'.format(
fields=fields, alias=target.name))
else:
query_fields.append(
'null AS {alias}'.format(alias=target.name))
return ', '.join(query_fields)
def from_table(table):
schema = tuple(
BigQuerySchemaField.from_schema_field(s)
for s in table.schema) if table.schema else None
return BigQueryTable(
table_id=table.table_id,
friendly_name=table.friendly_name,
description=table.description,
expires=table.expires,
partitioning_type=table.partitioning_type,
view_use_legacy_sql=table.view_use_legacy_sql
if table.view_query else None,
view_query=table.view_query if table.view_query else None,
schema=None if table.view_query else schema,
labels=table.labels)
def from_dict(value):
schema = value.get('schema', None)
if schema:
schema = tuple(BigQuerySchemaField.from_dict(s) for s in schema)
expires = value.get('expires', None)
if expires:
expires = parse_expires(expires)
return BigQueryTable(
table_id=value.get('table_id', None),
friendly_name=value.get('friendly_name', None),
description=value.get('description', None),
expires=expires,
partitioning_type=value.get('partitioning_type', None),
view_use_legacy_sql=value.get('view_use_legacy_sql', None),
view_query=value.get('view_query', None),
schema=schema,
labels=value.get('labels', None),
)
def to_table(dataset_ref, model):
schema = model.schema
if schema:
schema = tuple(
BigQuerySchemaField.to_schema_field(s) for s in schema)
else:
schema = None
table_ref = TableReference(dataset_ref, model.table_id)
table = Table(table_ref, schema)
table.friendly_name = model.friendly_name
table.description = model.description
table.expires = model.expires
table.partitioning_type = model.partitioning_type
if model.view_use_legacy_sql is not None:
table.view_use_legacy_sql = model.view_use_legacy_sql
if model.view_query is not None:
table.view_query = model.view_query
table.labels = model.labels if model.labels is not None else dict()
return table
def test_from_dict(self):
expected_schema_field1 = BigQuerySchemaField(
name='test',
field_type='STRING',
mode='NULLABLE',
description='test_description')
actual_schema_field1_1 = BigQuerySchemaField.from_dict({
'name':
'test',
'field_type':
'STRING',
'mode':
'NULLABLE',
'description':
'test_description'
})
self.assertEqual(expected_schema_field1, actual_schema_field1_1)
actual_schema_field1_2 = BigQuerySchemaField.from_dict({
'name':
'test',
'field_type':
'INTEGER',
'mode':
'REQUIRED',
'description':
'foo_bar'
})
self.assertNotEqual(expected_schema_field1, actual_schema_field1_2)
expected_schema_field2 = BigQuerySchemaField(
name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(expected_schema_field1, ))
actual_schema_field2_1 = BigQuerySchemaField.from_dict({
'name':
'test',
'field_type':
'RECORD',
'mode':
'NULLABLE',
'description':
'test_description',
'fields': [{
'name': 'test',
'field_type': 'STRING',
'mode': 'NULLABLE',
'description': 'test_description'
}]
})
self.assertEqual(expected_schema_field2, actual_schema_field2_1)
actual_schema_field2_2 = BigQuerySchemaField.from_dict({
'name':
'test',
'field_type':
'RECORD',
'mode':
'NULLABLE',
'description':
'test_description',
'fields': [{
'name': 'test',
'field_type': 'INTEGER',
'mode': 'REQUIRED',
'description': 'foo_bar'
}]
})
self.assertNotEqual(expected_schema_field2, actual_schema_field2_2)
def test_normalize_field_type(self):
self.assertEqual('INT64',
BigQuerySchemaField.normalize_field_type('INTEGER'))
self.assertEqual('INT64',
BigQuerySchemaField.normalize_field_type('INT64'))
self.assertEqual('FLOAT64',
BigQuerySchemaField.normalize_field_type('FLOAT'))
self.assertEqual('FLOAT64',
BigQuerySchemaField.normalize_field_type('FLOAT64'))
self.assertEqual('BOOL',
BigQuerySchemaField.normalize_field_type('BOOLEAN'))
self.assertEqual('BOOL',
BigQuerySchemaField.normalize_field_type('BOOL'))
self.assertEqual('BYTES',
BigQuerySchemaField.normalize_field_type('BYTES'))
self.assertEqual('DATE',
BigQuerySchemaField.normalize_field_type('DATE'))
self.assertEqual('DATETIME',
BigQuerySchemaField.normalize_field_type('DATETIME'))
self.assertEqual('TIME',
BigQuerySchemaField.normalize_field_type('TIME'))
self.assertEqual('TIMESTAMP',
BigQuerySchemaField.normalize_field_type('TIMESTAMP'))
self.assertEqual('STRING',
BigQuerySchemaField.normalize_field_type('STRING'))
self.assertRaises(
ValueError,
lambda: BigQuerySchemaField.normalize_field_type('DECIMAL'))
def test_eq(self):
schema_field1_1 = BigQuerySchemaField(name='test',
field_type='STRING',
mode='NULLABLE',
description='test_description')
schema_field1_2 = BigQuerySchemaField(name='test',
field_type='STRING',
mode='NULLABLE',
description='test_description')
self.assertEqual(schema_field1_1, schema_field1_2)
schema_field2_1 = BigQuerySchemaField(name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description')
schema_field2_2 = BigQuerySchemaField(name='test',
field_type='STRING',
mode='REQUIRED',
description='foo_bar')
self.assertNotEqual(schema_field2_1, schema_field2_2)
schema_field3_1 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(schema_field1_1, ))
schema_field3_2 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(schema_field1_1, ))
schema_field3_3 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=[schema_field1_1])
self.assertEqual(schema_field3_1, schema_field3_2)
self.assertEqual(schema_field3_1, schema_field3_3)
schema_field4_1 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(schema_field1_1, ))
schema_field4_2 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(schema_field1_1,
schema_field2_1))
schema_field4_3 = BigQuerySchemaField(name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(schema_field2_1,
schema_field1_1))
schema_field4_4 = BigQuerySchemaField(
name='test',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=[schema_field2_1, schema_field1_1])
self.assertNotEqual(schema_field4_1, schema_field4_2)
self.assertNotEqual(schema_field4_1, schema_field4_3)
self.assertNotEqual(schema_field4_1, schema_field4_4)
self.assertEqual(schema_field4_2, schema_field4_3)
self.assertEqual(schema_field4_2, schema_field4_4)
def test_to_table(self):
project = 'test'
dataset_ref = DatasetReference(project, 'test')
expected_table1 = make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description')
actual_table1_1 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description'))
self.assertEqual(expected_table1.table_id, actual_table1_1.table_id)
self.assertEqual(expected_table1.friendly_name,
actual_table1_1.friendly_name)
self.assertEqual(expected_table1.description,
actual_table1_1.description)
actual_table1_2 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='aaa',
friendly_name='foo_bar',
description='fizz_buzz'))
self.assertNotEqual(expected_table1.table_id, actual_table1_2.table_id)
self.assertNotEqual(expected_table1.friendly_name,
actual_table1_2.friendly_name)
self.assertNotEqual(expected_table1.description,
actual_table1_2.description)
expected_table2 = make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2018,
1,
1,
0,
0,
0,
tzinfo=UTC),
partitioning_type='DAY')
actual_table2_1 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type='DAY'))
self.assertEqual(expected_table2.expires, actual_table2_1.expires)
self.assertEqual(expected_table2.partitioning_type,
actual_table2_1.partitioning_type)
actual_table2_2 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='description',
expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type=None))
self.assertNotEqual(expected_table2.expires, actual_table2_2.expires)
self.assertNotEqual(expected_table2.partitioning_type,
actual_table2_2.partitioning_type)
expected_table3 = make_table(
project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test')
actual_table3_1 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test'))
self.assertEqual(expected_table3.view_use_legacy_sql,
actual_table3_1.view_use_legacy_sql)
self.assertEqual(expected_table3.view_query,
actual_table3_1.view_query)
actual_table3_2 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=True,
view_query='SELECT * FROM bigquery_datasetmanager.foo.bar'))
self.assertNotEqual(expected_table3.view_use_legacy_sql,
actual_table3_2.view_use_legacy_sql)
self.assertNotEqual(expected_table3.view_query,
actual_table3_2.view_query)
expected_table4 = make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(SchemaField(
name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'), ))
actual_table4_1 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(BigQuerySchemaField('test', 'INTEGER',
'NULLABLE',
'test_description',
None), )))
self.assertEqual(expected_table4.schema, actual_table4_1.schema)
actual_table4_2 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(SchemaField(name='test',
field_type='STRING',
mode='REQUIRED',
description='foo_bar'), )))
self.assertNotEqual(expected_table4.schema, actual_table4_2.schema)
expected_table5 = make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={'foo': 'bar'})
actual_table5_1 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={'foo': 'bar'}))
self.assertEqual(expected_table5.labels, actual_table5_1.labels)
actual_table5_2 = BigQueryTable.to_table(
dataset_ref,
BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={
'foo': 'bar',
'fizz': 'buzz'
}))
self.assertNotEqual(expected_table5.labels, actual_table5_2.labels)
def test_from_table(self):
project = 'test'
expected_table1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
actual_table1_1 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description'))
self.assertEqual(expected_table1, actual_table1_1)
actual_table1_2 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='foo_bar',
description='fizz_buzz'))
self.assertNotEqual(expected_table1, actual_table1_2)
expected_table2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2018,
1,
1,
0,
0,
0,
tzinfo=UTC),
partitioning_type='DAY')
actual_table2_1 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type='DAY'))
self.assertEqual(expected_table2, actual_table2_1)
actual_table2_2 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type=None))
self.assertNotEqual(expected_table2, actual_table2_2)
expected_table3 = BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test')
actual_table3_1 = BigQueryTable.from_table(
make_table(
project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test'))
self.assertEqual(expected_table3, actual_table3_1)
actual_table3_2 = BigQueryTable.from_table(
make_table(
project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=True,
view_query='SELECT * FROM bigquery_datasetmanager.foo.bar'))
self.assertNotEqual(expected_table3, actual_table3_2)
expected_table4 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(BigQuerySchemaField(
name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'), ),
labels=None)
actual_table4_1 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(SchemaField(name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'), )))
self.assertEqual(expected_table4, actual_table4_1)
actual_table4_2 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(SchemaField(name='test',
field_type='STRING',
mode='REQUIRED',
description='foo_bar'), )))
self.assertNotEqual(expected_table4, actual_table4_2)
expected_table5 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={'foo': 'bar'})
actual_table5_1 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={'foo': 'bar'}))
self.assertEqual(expected_table5, actual_table5_1)
actual_table5_2 = BigQueryTable.from_table(
make_table(project=project,
dataset_id='test',
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={
'foo': 'bar',
'fizz': 'buzz'
}))
self.assertNotEqual(expected_table5, actual_table5_2)
def test_eq(self):
table1_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
table1_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
self.assertEqual(table1_1, table1_2)
table2_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
table2_2 = BigQueryTable(
table_id='test',
friendly_name='fizz_buzz',
description='foo_bar',
expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type='DAY',
)
self.assertNotEqual(table2_1, table2_2)
table3_1 = BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
)
table3_2 = BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test')
self.assertNotEqual(table3_1, table3_2)
schema_field1 = BigQuerySchemaField(name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description')
schema_field2 = BigQuerySchemaField(name='test',
field_type='STRING',
mode='REQUIRED',
description='foo_bar')
table4_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, ))
table4_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, ))
self.assertEqual(table4_1, table4_2)
table5_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, ))
table5_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field2, ))
self.assertNotEqual(table5_1, table5_2)
table6_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, schema_field2))
table6_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field2, schema_field1))
table6_3 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=[schema_field1, schema_field2])
self.assertEqual(table6_1, table6_2)
self.assertEqual(table6_1, table6_3)
self.assertEqual(table6_2, table6_3)
table7_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
table7_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, ))
table7_3 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, schema_field2))
self.assertNotEqual(table7_1, table7_2)
self.assertNotEqual(table7_1, table7_3)
self.assertNotEqual(table7_2, table7_3)
label1 = {'foo': 'bar'}
label2 = {'fizz': 'buzz'}
table8_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label1)
table8_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label1)
self.assertEqual(table8_1, table8_2)
table9_1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label1)
table9_2 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label2)
self.assertNotEqual(table9_1, table9_2)
def test_from_dict(self):
expected_table1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
actual_table1_1 = BigQueryTable.from_dict({
'table_id':
'test',
'friendly_name':
'test_friendly_name',
'description':
'test_description'
})
self.assertEqual(expected_table1, actual_table1_1)
actual_table1_2 = BigQueryTable.from_dict({
'table_id': 'test',
'friendly_name': 'fizz_buzz',
'description': 'foo_bar',
'expires': '2018-01-01T00:00:00.000000+00:00',
'partitioning_type': 'DAY'
})
self.assertNotEqual(expected_table1, actual_table1_2)
expected_table2 = BigQueryTable(
table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test')
actual_table2_1 = BigQueryTable.from_dict({
'table_id':
'test',
'friendly_name':
'test_friendly_name',
'description':
'test_description',
'view_use_legacy_sql':
False,
'view_query':
'SELECT * FROM bigquery_datasetmanager.test.test'
})
self.assertEqual(expected_table2, actual_table2_1)
actual_table2_2 = BigQueryTable.from_dict({
'table_id':
'test',
'friendly_name':
'test_friendly_name',
'description':
'test_description',
'view_use_legacy_sql':
True,
'view_query':
'SELECT * FROM bigquery_datasetmanager.foo.bar'
})
self.assertNotEqual(expected_table2, actual_table2_2)
expected_table3 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(BigQuerySchemaField(
'test', 'INTEGER', 'NULLABLE',
'test_description', None), ))
actual_table3_1 = BigQueryTable.from_dict({
'table_id':
'test',
'friendly_name':
'test_friendly_name',
'description':
'test_description',
'schema': [{
'name': 'test',
'field_type': 'INTEGER',
'mode': 'NULLABLE',
'description': 'test_description',
'fields': None
}]
})
self.assertEqual(expected_table3, actual_table3_1)
actual_table3_2 = BigQueryTable.from_dict({
'table_id':
'test',
'friendly_name':
'test_friendly_name',
'description':
'test_description',
'schema': [{
'name': 'test',
'field_type': 'STRING',
'mode': 'REQUIRED',
'description': 'foo_bar',
'fields': None
}]
})
self.assertNotEqual(expected_table3, actual_table3_2)
expected_table4 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels={'foo': 'bar'})
actual_table4_1 = BigQueryTable.from_dict({
'table_id': 'test',
'friendly_name': 'test_friendly_name',
'description': 'test_description',
'labels': {
'foo': 'bar'
}
})
self.assertEqual(expected_table4, actual_table4_1)
actual_table4_2 = BigQueryTable.from_dict({
'table_id': 'test',
'friendly_name': 'test_friendly_name',
'description': 'test_description',
'labels': {
'foo': 'bar',
'fizz': 'buzz'
}
})
self.assertNotEqual(expected_table4, actual_table4_2)
def test_dump_table(self):
table1 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description')
expected_dump_data1 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema: null
labels: null
"""
actual_dump_data1 = dump(table1)
self.assertEqual(expected_dump_data1, actual_dump_data1)
table2 = BigQueryTable(
table_id='test',
friendly_name='fizz_buzz',
description='foo_bar',
expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type='DAY',
)
expected_dump_data2 = """table_id: test
friendly_name: fizz_buzz
description: foo_bar
expires: 2018-01-01T00:00:00.000000+0000
partitioning_type: DAY
view_use_legacy_sql: null
view_query: null
schema: null
labels: null
"""
actual_dump_data2 = dump(table2)
self.assertEqual(expected_dump_data2, actual_dump_data2)
table3 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query="""SELECT * FROM
bigquery_datasetmanager.test.test""")
expected_dump_data3 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: false
view_query: |-
SELECT * FROM
bigquery_datasetmanager.test.test
schema: null
labels: null
"""
actual_dump_data3 = dump(table3)
self.assertEqual(expected_dump_data3, actual_dump_data3)
schema_field1 = BigQuerySchemaField(name='test1',
field_type='INTEGER',
mode='NULLABLE',
description='test_description')
schema_field2 = BigQuerySchemaField(name='test2',
field_type='STRING',
mode='REQUIRED',
description='foo_bar')
schema_field3 = BigQuerySchemaField(name='test3',
field_type='RECORD',
mode='NULLABLE',
description='fizz_buzz',
fields=(schema_field1,
schema_field2))
schema_field4 = BigQuerySchemaField(name='test4',
field_type='RECORD',
mode='NULLABLE',
description='aaa_bbb',
fields=(schema_field3, ))
table4 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, ))
expected_dump_data4 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema:
- name: test1
field_type: INTEGER
mode: NULLABLE
description: test_description
fields: null
labels: null
"""
actual_dump_data4 = dump(table4)
self.assertEqual(expected_dump_data4, actual_dump_data4)
table5 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, schema_field2))
expected_dump_data5 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema:
- name: test1
field_type: INTEGER
mode: NULLABLE
description: test_description
fields: null
- name: test2
field_type: STRING
mode: REQUIRED
description: foo_bar
fields: null
labels: null
"""
actual_dump_data5 = dump(table5)
self.assertEqual(expected_dump_data5, actual_dump_data5)
table6 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field3, ))
expected_dump_data6 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema:
- name: test3
field_type: RECORD
mode: NULLABLE
description: fizz_buzz
fields:
- name: test1
field_type: INTEGER
mode: NULLABLE
description: test_description
fields: null
- name: test2
field_type: STRING
mode: REQUIRED
description: foo_bar
fields: null
labels: null
"""
actual_dump_data6 = dump(table6)
self.assertEqual(expected_dump_data6, actual_dump_data6)
table7 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field4, ))
expected_dump_data7 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema:
- name: test4
field_type: RECORD
mode: NULLABLE
description: aaa_bbb
fields:
- name: test3
field_type: RECORD
mode: NULLABLE
description: fizz_buzz
fields:
- name: test1
field_type: INTEGER
mode: NULLABLE
description: test_description
fields: null
- name: test2
field_type: STRING
mode: REQUIRED
description: foo_bar
fields: null
labels: null
"""
actual_dump_data7 = dump(table7)
self.assertEqual(expected_dump_data7, actual_dump_data7)
label1 = {'foo': 'bar'}
label2 = {'fizz': 'buzz', 'aaa': 'bbb'}
table8 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label1)
expected_dump_data8 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema: null
labels:
foo: bar
"""
actual_dump_data8 = dump(table8)
self.assertEqual(expected_dump_data8, actual_dump_data8)
table9 = BigQueryTable(table_id='test',
friendly_name='test_friendly_name',
description='test_description',
labels=label2)
expected_dump_data9 = """table_id: test
friendly_name: test_friendly_name
description: test_description
expires: null
partitioning_type: null
view_use_legacy_sql: null
view_query: null
schema: null
labels:
aaa: bbb
fizz: buzz
"""
actual_dump_data9 = dump(table9)
self.assertEqual(expected_dump_data9, actual_dump_data9)
def test_get_change_tables(self):
schema_field1 = BigQuerySchemaField(
name='test',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'
)
schema_field2 = BigQuerySchemaField(
name='test',
field_type='STRING',
mode='REQUIRED',
description='foo_bar'
)
label1 = {
'foo': 'bar'
}
label2 = {
'fizz': 'buzz'
}
source_table1_1 = BigQueryTable(
table_id='test1',
friendly_name='test_friendly_name',
description='test_description'
)
source_table1_2 = BigQueryTable(
table_id='test2',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2018, 1, 1, 0, 0, 0, tzinfo=UTC),
partitioning_type='DAY'
)
source_table1_3 = BigQueryTable(
table_id='test3',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=False,
view_query='SELECT * FROM bigquery_datasetmanager.test.test'
)
source_table1_4 = BigQueryTable(
table_id='test4',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field1, )
)
source_table1_5 = BigQueryTable(
table_id='test5',
friendly_name='test_friendly_name',
description='test_description',
labels=label1
)
target_table1_1 = BigQueryTable(
table_id='test1',
friendly_name='foo_bar',
description='fizz_buzz'
)
target_table1_2 = BigQueryTable(
table_id='test2',
friendly_name='test_friendly_name',
description='test_description',
expires=datetime(2019, 1, 1, 0, 0, 0, tzinfo=UTC)
)
target_table1_3 = BigQueryTable(
table_id='test3',
friendly_name='test_friendly_name',
description='test_description',
view_use_legacy_sql=True,
view_query='SELECT * FROM bigquery_datasetmanager.foo.bar'
)
target_table1_4 = BigQueryTable(
table_id='test4',
friendly_name='test_friendly_name',
description='test_description',
schema=(schema_field2, )
)
target_table1_5 = BigQueryTable(
table_id='test5',
friendly_name='test_friendly_name',
description='test_description',
labels=label2
)
source1 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
target1 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
actual_count1, actual_results1 = TableAction.get_change_tables(source1, target1)
self.assertEqual(actual_count1, 5)
self.assertEqual(set(actual_results1),
{target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5})
source2 = [source_table1_5, source_table1_4, source_table1_3,
source_table1_2, source_table1_1]
target2 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
actual_count2, actual_results2 = TableAction.get_change_tables(source2, target2)
self.assertEqual(actual_count2, 5)
self.assertEqual(set(actual_results2),
{target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5})
source3 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
target3 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
actual_count3, actual_results3 = TableAction.get_change_tables(source3, target3)
self.assertEqual(actual_count3, 0)
self.assertEqual(actual_results3, ())
source4 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
target4 = [source_table1_5, source_table1_4, source_table1_3,
source_table1_2, source_table1_1]
actual_count4, actual_results4 = TableAction.get_change_tables(source4, target4)
self.assertEqual(actual_count4, 0)
self.assertEqual(actual_results4, ())
source5 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
target5 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
actual_count5, actual_results5 = TableAction.get_change_tables(source5, target5)
self.assertEqual(actual_count5, 0)
self.assertEqual(actual_results5, ())
source6 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
target6 = [target_table1_5, target_table1_4, target_table1_3,
target_table1_2, target_table1_1]
actual_count6, actual_results6 = TableAction.get_change_tables(source6, target6)
self.assertEqual(actual_count6, 0)
self.assertEqual(actual_results6, ())
source7 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
target7 = [target_table1_1, target_table1_2]
actual_count7, actual_results7 = TableAction.get_change_tables(source7, target7)
self.assertEqual(actual_count7, 2)
self.assertEqual(set(actual_results7), {target_table1_1, target_table1_2})
source8 = [source_table1_1, source_table1_2, source_table1_3,
source_table1_4, source_table1_5]
target8 = [target_table1_3, target_table1_4, target_table1_5]
actual_count8, actual_results8 = TableAction.get_change_tables(source8, target8)
self.assertEqual(actual_count8, 3)
self.assertEqual(set(actual_results8),
{target_table1_3, target_table1_4, target_table1_5})
source9 = [source_table1_1, source_table1_2]
target9 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
actual_count9, actual_results9 = TableAction.get_change_tables(source9, target9)
self.assertEqual(actual_count9, 2)
self.assertEqual(set(actual_results9), {target_table1_1, target_table1_2})
source10 = [source_table1_3, source_table1_4, source_table1_5]
target10 = [target_table1_1, target_table1_2, target_table1_3,
target_table1_4, target_table1_5]
actual_count10, actual_results10 = TableAction.get_change_tables(source10, target10)
self.assertEqual(actual_count10, 3)
self.assertEqual(set(actual_results10),
{target_table1_3, target_table1_4, target_table1_5})
def test_build_query_field(self):
source_schema_field1 = BigQuerySchemaField(
name='test1',
field_type='STRING',
mode='NULLABLE',
description='test_description'
)
source_schema_field2 = BigQuerySchemaField(
name='test2',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'
)
source_schema_field3 = BigQuerySchemaField(
name='test3',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(
BigQuerySchemaField(
name='foo_bar',
field_type='STRING',
mode='NULLABLE',
description='test_description'
),
)
)
source_schema_field4 = BigQuerySchemaField(
name='test4',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(
BigQuerySchemaField(
name='fizz',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'
),
BigQuerySchemaField(
name='buzz',
field_type='BOOL',
mode='NULLABLE',
description='test_description'
),
)
)
target_schema_field1 = BigQuerySchemaField(
name='test1',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'
)
target_schema_field2 = BigQuerySchemaField(
name='test2',
field_type='STRING',
mode='NULLABLE',
description='test_description'
)
target_schema_field3 = BigQuerySchemaField(
name='test3',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(
BigQuerySchemaField(
name='foo_bar',
field_type='INTEGER',
mode='NULLABLE',
description='test_description'
),
)
)
target_schema_field4 = BigQuerySchemaField(
name='test4',
field_type='RECORD',
mode='NULLABLE',
description='test_description',
fields=(
BigQuerySchemaField(
name='fizz',
field_type='FLOAT',
mode='NULLABLE',
description='test_description'
),
BigQuerySchemaField(
name='buzz',
field_type='STRING',
mode='NULLABLE',
description='test_description'
),
)
)
source1 = (source_schema_field1, source_schema_field2,
source_schema_field3, source_schema_field4)
target1 = (target_schema_field1, target_schema_field2,
target_schema_field3, target_schema_field4)
expected_query_field = 'cast(test1 AS INT64) AS test1, ' \
'cast(test2 AS STRING) AS test2, ' \
'struct(cast(test3.foo_bar AS INT64) AS foo_bar) AS test3, ' \
'struct(cast(test4.fizz AS FLOAT64) AS fizz, ' \
'cast(test4.buzz AS STRING) AS buzz) AS test4'
actual_query_field = TableAction.build_query_field(source1, target1)
self.assertEqual(expected_query_field, actual_query_field)