def test_merge_array_schemas_records(self): schemas = [ bigquery_schema.translate_json_to_schema( {'field1': [{ 'nested1': 'value1' }]}), bigquery_schema.translate_json_to_schema( {'field1': [{ 'nested2': 'value1' }]}) ] merged_schema = bigquery_schema.merge_schemas(schemas) self.assertEqual(merged_schema, [{ 'name': 'field1', 'field_type': 'RECORD', 'mode': 'REPEATED', 'fields': [{ 'name': 'nested1', 'field_type': 'STRING', 'mode': 'NULLABLE' }, { 'name': 'nested2', 'field_type': 'STRING', 'mode': 'NULLABLE' }] }])
def test_merge_schemas_records(self): schemas = [ bigquery_schema.translate_json_to_schema( {'recordField': { 'field1': 'string' }}), bigquery_schema.translate_json_to_schema( {'recordfield': { 'field1': 'string', 'field2': [2] }}) ] merged_schema = bigquery_schema.merge_schemas(schemas) self.assertEqual(merged_schema, [{ 'name': 'recordField', 'field_type': 'RECORD', 'mode': 'NULLABLE', 'fields': [{ 'name': 'field1', 'field_type': 'STRING', 'mode': 'NULLABLE' }, { 'name': 'field2', 'field_type': 'NUMERIC', 'mode': 'REPEATED' }] }])
def test_merge_schemas_records(self): schemas = [ bigquery_schema.translate_json_to_schema( {'record_field': { 'field1': 'string' }}), bigquery_schema.translate_json_to_schema( {'record_field': { 'field1': 'string', 'field2': [2] }}) ] merged_schema = bigquery_schema.merge_schemas(schemas) self.assertEqual(len(merged_schema), 1) record_field = merged_schema[0] self.assertEqual(record_field.field_type, 'RECORD') self.assertEqual(len(record_field.fields), 2) _, field1 = bigquery_schema._get_field_by_name(record_field.fields, 'field1') self.assertEqual(field1.field_type, 'STRING') self.assertEqual(field1.mode, 'NULLABLE') _, field2 = bigquery_schema._get_field_by_name(record_field.fields, 'field2') self.assertEqual(field2.field_type, 'NUMERIC') self.assertEqual(field2.mode, 'REPEATED')
def test_merge_schemas_basic(self): schemas = [ bigquery_schema.translate_json_to_schema({'field1': 'string'}), bigquery_schema.translate_json_to_schema({'field2': 3}) ] merged_schema = bigquery_schema.merge_schemas(schemas) self.assertEqual(len(merged_schema), 2) _, field1 = bigquery_schema._get_field_by_name(merged_schema, 'field1') self.assertEqual(field1.field_type, 'STRING') _, field2 = bigquery_schema._get_field_by_name(merged_schema, 'field2') self.assertEqual(field2.field_type, 'NUMERIC')
def test_merge_schemas_basic(self): schemas = [ bigquery_schema.translate_json_to_schema({'field1': 'string'}), bigquery_schema.translate_json_to_schema({'field2': 3}) ] merged_schema = bigquery_schema.merge_schemas(schemas) self.assertEqual(merged_schema, [ { 'name': 'field1', 'field_type': 'STRING', 'mode': 'NULLABLE' }, { 'name': 'field2', 'field_type': 'NUMERIC', 'mode': 'NULLABLE' }, ])
def test_timestamp(self): document = {'timestamp': '2019-01-01T00:01:00'} schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual(schema, [ { 'name': 'timestamp', 'field_type': 'STRING', 'mode': 'NULLABLE' }, ])
def test_numeric(self): document = {'integer_field': 111, 'float_field': 22.0} schema = bigquery_schema.translate_json_to_schema( document) self.assertEqual(schema, [{'name': 'integer_field', 'field_type': 'NUMERIC', 'mode': 'NULLABLE'}, {'name': 'float_field', 'field_type': 'NUMERIC', 'mode': 'NULLABLE'}, ])
def test_bool(self): document = {'bool_array_field': [True, False], 'bool_field': False} schema = bigquery_schema.translate_json_to_schema( document) self.assertEqual(schema, [{'name': 'bool_array_field', 'field_type': 'BOOL', 'mode': 'REPEATED'}, {'name': 'bool_field', 'field_type': 'BOOL', 'mode': 'NULLABLE'} ])
def test_array(self): document = {'array_field': [{'string_field': 'string_value'}]} schema = bigquery_schema.translate_json_to_schema( document) self.assertEqual(schema, [{'name': 'array_field', 'field_type': 'RECORD', 'mode': 'REPEATED', 'fields': [ {'name': 'string_field', 'field_type': 'STRING', 'mode': 'NULLABLE' }]}])
def test_array(self): document = {'array_field': [{'string_field': 'string_value'}]} schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual(len(schema), 1) array_field = schema[0] self.assertEqual(array_field.name, 'array_field') self.assertEqual(array_field.field_type, 'RECORD') self.assertEqual(array_field.mode, 'REPEATED') self.assertEqual(len(array_field.fields), 1) string_field = array_field.fields[0] self.assertEqual(string_field.name, 'string_field') self.assertEqual(string_field.field_type, 'STRING') self.assertEqual(string_field.mode, 'NULLABLE')
def test_record(self): document = {'record_field': {'string_field': 'string_value'}} schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual(len(schema), 1) record_field = schema[0] self.assertEqual(record_field.name, 'record_field') self.assertEqual(record_field.field_type, 'RECORD') self.assertEqual(record_field.mode, 'NULLABLE') self.assertEqual(len(record_field.fields), 1) string_field = record_field.fields[0] self.assertEqual(string_field.name, 'string_field') self.assertEqual(string_field.field_type, 'STRING') self.assertEqual(string_field.mode, 'NULLABLE')
def test_numeric(self): document = {'integer_field': 111, 'float_field': 22.0} schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual(len(schema), 2) _, integer_field = bigquery_schema._get_field_by_name( schema, 'integer_field') self.assertEqual(integer_field.name, 'integer_field') self.assertEqual(integer_field.field_type, 'NUMERIC') self.assertEqual(integer_field.mode, 'NULLABLE') _, float_field = bigquery_schema._get_field_by_name( schema, 'float_field') self.assertEqual(float_field.name, 'float_field') self.assertEqual(float_field.field_type, 'NUMERIC') self.assertEqual(float_field.mode, 'NULLABLE')
def test_bool(self): document = {'bool_array_field': [True, False], 'bool_field': False} schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual(len(schema), 2) _, bool_array_field = bigquery_schema._get_field_by_name( schema, 'bool_array_field') self.assertEqual(bool_array_field.name, 'bool_array_field') self.assertEqual(bool_array_field.field_type, 'BOOL') self.assertEqual(bool_array_field.mode, 'REPEATED') _, bool_field = bigquery_schema._get_field_by_name( schema, 'bool_field') self.assertEqual(bool_field.name, 'bool_field') self.assertEqual(bool_field.field_type, 'BOOL') self.assertEqual(bool_field.mode, 'NULLABLE')
def test_merge_array_schemas_records(self): schema = bigquery_schema.translate_json_to_schema([{ 'field1': 'value1' }, { 'field2': 'value1' }]) self.assertEqual(len(schema), 2) fields_found = [False, False] for field in schema: if field.name == 'field1': fields_found[0] = True if field.name == 'field2': fields_found[1] = True assert field.field_type == 'STRING' self.assertTrue(fields_found[0] and fields_found[1])
def test_addtional_properties_merge_schema_simple(self): rest_schema = [{ 'name': 'property_1', 'field_type': 'STRING', 'description': 'description-1', 'mode': 'NULLABLE' }, { 'name': 'property_2', 'field_type': 'RECORD', 'description': 'description-2', 'mode': 'REPEATED', 'fields': [{ 'name': 'name', 'field_type': 'STRING', 'description': 'additionalProperties name', 'mode': 'NULLABLE' }, { 'name': 'value', 'field_type': 'STRING', 'description': 'description-2.', 'mode': 'NULLABLE' }] }] document = { 'property_1': 'value_1', 'property_2': { 'add_prop_1': 'add_value_1', 'add_prop_2': 'add_value_2' }, 'property_3': 'value_3' } document_schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual( bigquery_schema.merge_schemas([rest_schema, document_schema]), rest_schema + [{ 'name': 'property_3', 'field_type': 'STRING', 'mode': 'NULLABLE' }])
def test_addtional_properties_merge_schema_object(self): self.maxDiff = None rest_schema = [{ 'name': 'property_1', 'field_type': 'STRING', 'description': 'description-1', 'mode': 'NULLABLE' }, { 'name': 'property_2', 'field_type': 'RECORD', 'description': 'description-2', 'mode': 'REPEATED', 'fields': [{ 'name': 'name', 'field_type': 'STRING', 'description': 'additionalProperties name', 'mode': 'NULLABLE' }, { 'name': 'value', 'field_type': 'RECORD', 'mode': 'NULLABLE' }] }] document = { 'property_1': 'value_1', 'property_2': { 'add_prop_1': { 'key_1': 1 }, 'add_prop_2': { 'key_1': 2 } }, 'property_3': 'value_3' } document_schema = bigquery_schema.translate_json_to_schema(document) self.assertEqual( bigquery_schema.merge_schemas([rest_schema, document_schema]), [{ 'name': 'property_1', 'field_type': 'STRING', 'description': 'description-1', 'mode': 'NULLABLE' }, { 'name': 'property_2', 'field_type': 'RECORD', 'description': 'description-2', 'mode': 'REPEATED', 'fields': [{ 'name': 'name', 'field_type': 'STRING', 'description': 'additionalProperties name', 'mode': 'NULLABLE' }, { 'name': 'value', 'field_type': 'RECORD', 'mode': 'NULLABLE', 'fields': [{ 'name': 'key_1', 'field_type': 'NUMERIC', 'mode': 'NULLABLE' }] }] }, { 'name': 'property_3', 'field_type': 'STRING', 'mode': 'NULLABLE' }])
def add_input(self, schema, element): resource_schema = self.element_to_schema(element) json_schema = bigquery_schema.translate_json_to_schema(element) return bigquery_schema.merge_schemas( [schema, resource_schema, json_schema])
def add_input(self, schema, element): new_schema = bigquery_schema.translate_json_to_schema(element) return bigquery_schema.merge_schemas([schema, new_schema])