def test_merge_array_schemas_records(self):
schemas = [
bigquery_schema.translate_json_to_schema(
{'field1': [{
'nested1': 'value1'
}]}),
bigquery_schema.translate_json_to_schema(
{'field1': [{
'nested2': 'value1'
}]})
]
merged_schema = bigquery_schema.merge_schemas(schemas)
self.assertEqual(merged_schema, [{
'name':
'field1',
'field_type':
'RECORD',
'mode':
'REPEATED',
'fields': [{
'name': 'nested1',
'field_type': 'STRING',
'mode': 'NULLABLE'
}, {
'name': 'nested2',
'field_type': 'STRING',
'mode': 'NULLABLE'
}]
}])
def test_merge_schemas_records(self):
schemas = [
bigquery_schema.translate_json_to_schema(
{'recordField': {
'field1': 'string'
}}),
bigquery_schema.translate_json_to_schema(
{'recordfield': {
'field1': 'string',
'field2': [2]
}})
]
merged_schema = bigquery_schema.merge_schemas(schemas)
self.assertEqual(merged_schema, [{
'name':
'recordField',
'field_type':
'RECORD',
'mode':
'NULLABLE',
'fields': [{
'name': 'field1',
'field_type': 'STRING',
'mode': 'NULLABLE'
}, {
'name': 'field2',
'field_type': 'NUMERIC',
'mode': 'REPEATED'
}]
}])
def test_merge_schemas_records(self):
schemas = [
bigquery_schema.translate_json_to_schema(
{'record_field': {
'field1': 'string'
}}),
bigquery_schema.translate_json_to_schema(
{'record_field': {
'field1': 'string',
'field2': [2]
}})
]
merged_schema = bigquery_schema.merge_schemas(schemas)
self.assertEqual(len(merged_schema), 1)
record_field = merged_schema[0]
self.assertEqual(record_field.field_type, 'RECORD')
self.assertEqual(len(record_field.fields), 2)
_, field1 = bigquery_schema._get_field_by_name(record_field.fields,
'field1')
self.assertEqual(field1.field_type, 'STRING')
self.assertEqual(field1.mode, 'NULLABLE')
_, field2 = bigquery_schema._get_field_by_name(record_field.fields,
'field2')
self.assertEqual(field2.field_type, 'NUMERIC')
self.assertEqual(field2.mode, 'REPEATED')
def test_merge_schemas_basic(self):
schemas = [
bigquery_schema.translate_json_to_schema({'field1': 'string'}),
bigquery_schema.translate_json_to_schema({'field2': 3})
]
merged_schema = bigquery_schema.merge_schemas(schemas)
self.assertEqual(len(merged_schema), 2)
_, field1 = bigquery_schema._get_field_by_name(merged_schema, 'field1')
self.assertEqual(field1.field_type, 'STRING')
_, field2 = bigquery_schema._get_field_by_name(merged_schema, 'field2')
self.assertEqual(field2.field_type, 'NUMERIC')
def test_merge_schemas_basic(self):
schemas = [
bigquery_schema.translate_json_to_schema({'field1': 'string'}),
bigquery_schema.translate_json_to_schema({'field2': 3})
]
merged_schema = bigquery_schema.merge_schemas(schemas)
self.assertEqual(merged_schema, [
{
'name': 'field1',
'field_type': 'STRING',
'mode': 'NULLABLE'
},
{
'name': 'field2',
'field_type': 'NUMERIC',
'mode': 'NULLABLE'
},
])
def test_timestamp(self):
document = {'timestamp': '2019-01-01T00:01:00'}
schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(schema, [
{
'name': 'timestamp',
'field_type': 'STRING',
'mode': 'NULLABLE'
},
])
def test_numeric(self):
document = {'integer_field': 111, 'float_field': 22.0}
schema = bigquery_schema.translate_json_to_schema(
document)
self.assertEqual(schema, [{'name': 'integer_field',
'field_type': 'NUMERIC',
'mode': 'NULLABLE'},
{'name': 'float_field',
'field_type': 'NUMERIC',
'mode': 'NULLABLE'},
])
def test_bool(self):
document = {'bool_array_field': [True, False], 'bool_field': False}
schema = bigquery_schema.translate_json_to_schema(
document)
self.assertEqual(schema, [{'name': 'bool_array_field',
'field_type': 'BOOL',
'mode': 'REPEATED'},
{'name': 'bool_field',
'field_type': 'BOOL',
'mode': 'NULLABLE'}
])
def test_array(self):
document = {'array_field': [{'string_field': 'string_value'}]}
schema = bigquery_schema.translate_json_to_schema(
document)
self.assertEqual(schema, [{'name': 'array_field',
'field_type': 'RECORD',
'mode': 'REPEATED',
'fields': [
{'name': 'string_field',
'field_type': 'STRING',
'mode': 'NULLABLE'
}]}])
def test_array(self):
document = {'array_field': [{'string_field': 'string_value'}]}
schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(len(schema), 1)
array_field = schema[0]
self.assertEqual(array_field.name, 'array_field')
self.assertEqual(array_field.field_type, 'RECORD')
self.assertEqual(array_field.mode, 'REPEATED')
self.assertEqual(len(array_field.fields), 1)
string_field = array_field.fields[0]
self.assertEqual(string_field.name, 'string_field')
self.assertEqual(string_field.field_type, 'STRING')
self.assertEqual(string_field.mode, 'NULLABLE')
def test_record(self):
document = {'record_field': {'string_field': 'string_value'}}
schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(len(schema), 1)
record_field = schema[0]
self.assertEqual(record_field.name, 'record_field')
self.assertEqual(record_field.field_type, 'RECORD')
self.assertEqual(record_field.mode, 'NULLABLE')
self.assertEqual(len(record_field.fields), 1)
string_field = record_field.fields[0]
self.assertEqual(string_field.name, 'string_field')
self.assertEqual(string_field.field_type, 'STRING')
self.assertEqual(string_field.mode, 'NULLABLE')
def test_numeric(self):
document = {'integer_field': 111, 'float_field': 22.0}
schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(len(schema), 2)
_, integer_field = bigquery_schema._get_field_by_name(
schema, 'integer_field')
self.assertEqual(integer_field.name, 'integer_field')
self.assertEqual(integer_field.field_type, 'NUMERIC')
self.assertEqual(integer_field.mode, 'NULLABLE')
_, float_field = bigquery_schema._get_field_by_name(
schema, 'float_field')
self.assertEqual(float_field.name, 'float_field')
self.assertEqual(float_field.field_type, 'NUMERIC')
self.assertEqual(float_field.mode, 'NULLABLE')
def test_bool(self):
document = {'bool_array_field': [True, False], 'bool_field': False}
schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(len(schema), 2)
_, bool_array_field = bigquery_schema._get_field_by_name(
schema, 'bool_array_field')
self.assertEqual(bool_array_field.name, 'bool_array_field')
self.assertEqual(bool_array_field.field_type, 'BOOL')
self.assertEqual(bool_array_field.mode, 'REPEATED')
_, bool_field = bigquery_schema._get_field_by_name(
schema, 'bool_field')
self.assertEqual(bool_field.name, 'bool_field')
self.assertEqual(bool_field.field_type, 'BOOL')
self.assertEqual(bool_field.mode, 'NULLABLE')
def test_merge_array_schemas_records(self):
schema = bigquery_schema.translate_json_to_schema([{
'field1': 'value1'
}, {
'field2': 'value1'
}])
self.assertEqual(len(schema), 2)
fields_found = [False, False]
for field in schema:
if field.name == 'field1':
fields_found[0] = True
if field.name == 'field2':
fields_found[1] = True
assert field.field_type == 'STRING'
self.assertTrue(fields_found[0] and fields_found[1])
def test_addtional_properties_merge_schema_simple(self):
rest_schema = [{
'name': 'property_1',
'field_type': 'STRING',
'description': 'description-1',
'mode': 'NULLABLE'
}, {
'name':
'property_2',
'field_type':
'RECORD',
'description':
'description-2',
'mode':
'REPEATED',
'fields': [{
'name': 'name',
'field_type': 'STRING',
'description': 'additionalProperties name',
'mode': 'NULLABLE'
}, {
'name': 'value',
'field_type': 'STRING',
'description': 'description-2.',
'mode': 'NULLABLE'
}]
}]
document = {
'property_1': 'value_1',
'property_2': {
'add_prop_1': 'add_value_1',
'add_prop_2': 'add_value_2'
},
'property_3': 'value_3'
}
document_schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(
bigquery_schema.merge_schemas([rest_schema, document_schema]),
rest_schema + [{
'name': 'property_3',
'field_type': 'STRING',
'mode': 'NULLABLE'
}])
def test_addtional_properties_merge_schema_object(self):
self.maxDiff = None
rest_schema = [{
'name': 'property_1',
'field_type': 'STRING',
'description': 'description-1',
'mode': 'NULLABLE'
}, {
'name':
'property_2',
'field_type':
'RECORD',
'description':
'description-2',
'mode':
'REPEATED',
'fields': [{
'name': 'name',
'field_type': 'STRING',
'description': 'additionalProperties name',
'mode': 'NULLABLE'
}, {
'name': 'value',
'field_type': 'RECORD',
'mode': 'NULLABLE'
}]
}]
document = {
'property_1': 'value_1',
'property_2': {
'add_prop_1': {
'key_1': 1
},
'add_prop_2': {
'key_1': 2
}
},
'property_3': 'value_3'
}
document_schema = bigquery_schema.translate_json_to_schema(document)
self.assertEqual(
bigquery_schema.merge_schemas([rest_schema, document_schema]),
[{
'name': 'property_1',
'field_type': 'STRING',
'description': 'description-1',
'mode': 'NULLABLE'
}, {
'name':
'property_2',
'field_type':
'RECORD',
'description':
'description-2',
'mode':
'REPEATED',
'fields': [{
'name': 'name',
'field_type': 'STRING',
'description': 'additionalProperties name',
'mode': 'NULLABLE'
}, {
'name':
'value',
'field_type':
'RECORD',
'mode':
'NULLABLE',
'fields': [{
'name': 'key_1',
'field_type': 'NUMERIC',
'mode': 'NULLABLE'
}]
}]
}, {
'name': 'property_3',
'field_type': 'STRING',
'mode': 'NULLABLE'
}])
def add_input(self, schema, element):
resource_schema = self.element_to_schema(element)
json_schema = bigquery_schema.translate_json_to_schema(element)
return bigquery_schema.merge_schemas(
[schema, resource_schema, json_schema])
def add_input(self, schema, element):
new_schema = bigquery_schema.translate_json_to_schema(element)
return bigquery_schema.merge_schemas([schema, new_schema])
