def testRecordWithMap(self): filepath = self.directory + '/RecordWithMap.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # should have 1 field self.assertEqual( len(obj.children['records'].files['RecordWithMap'].fields), 4 ) # field should be of type map self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['thingMap'].fieldtype, # NOQA 'map' ) # thingmap field should be of type Thing self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['thingMap'].map_type.reference_name, # NOQA 'Thing' ) # map int field should be of type Thing self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['intMap'].map_type.avrotype, # NOQA 'int' )
def testRecordWithNestedMap(self): filepath = self.directory + '/RecordWithNestedMap.avsc' reader = AvscReader(file=filepath) reader.read() # test parsing works for nested maps obj = reader.file_tree file = obj.children['records'].files['RecordWithNestedMap'] self.assertEqual( len(obj.children['records'].files['RecordWithNestedMap'].fields), 4 ) # assert reader picked up the nested map self.assertEqual( file.fields['nestedThingMap'].map_type.map_type.name, 'Thing' ) self.assertEqual( file.fields['nestedThingMap'].map_type.map_type.fieldtype, 'reference' ) # assert reader picked up nested array in map self.assertEqual( file.fields['mappedThingArray'].map_type.array_item_type.name, 'Thing' ) self.assertEqual( file.fields['mappedThingArray'].map_type.array_item_type.fieldtype, 'reference' )
def testRecordWithArray(self): filepath = self.directory + '/RecordWithArray.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # should have 3 fields self.assertEqual( len(obj.children['records'].files['RecordWithArray'].fields), 3 ) # field 0 should be of type reference self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['things'].array_item_type.fieldtype, # NOQA 'reference' ) # field 1 should be of type primitive self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['numbers'].array_item_type.fieldtype, # NOQA 'primitive' ) # field 2 should be of type reference self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['things2'].array_item_type.fieldtype, # NOQA 'reference' )
def testReadThing(self): filepath = self.directory + '/Thing.avsc' reader = AvscReader(file=filepath) reader.read() # test node tree structure is correct obj = reader.file_tree self.assertEqual( obj.name, '' ) self.assertEqual( list(obj.children.keys()), ['records'] ) # test Thing file is in correct place self.assertEqual( obj.children['records'].files['Thing'].name, 'Thing' ) # test Thing.id exists and only once self.assertEqual( len(obj.children['records'].files['Thing'].fields), 1 ) # test thing.id field has necessary information self.assertEqual( obj.children['records'].files['Thing'].fields['id'].avrotype, 'int' )
def test_namespace_duplicate(self): if os.path.isdir("tests/avsc/namespace_duplicate_test/test/"): shutil.rmtree("tests/avsc/namespace_duplicate_test/test/") reader = AvscReader( directory=self.source) reader.read() writer = AvroWriter( reader.file_tree, pip=None, author=None, package_version=None ) writer.write(root_dir="tests/avsc/namespace_duplicate_test/test/") del reader self.assertFalse( os.path.isfile("tests/avsc/namespace_duplicate_test/test/namespace_duplicate_test/unique/Common.py") ) self.assertTrue( os.path.isfile("tests/avsc/namespace_duplicate_test/test/namespace_duplicate_test/common/enums/CommonEnum.py") ) self.assertTrue( os.path.isfile("tests/avsc/namespace_duplicate_test/test/namespace_duplicate_test/unique/UniqueEnum.py") ) self.assertTrue( os.path.isfile("tests/avsc/namespace_duplicate_test/test/namespace_duplicate_test/common/TraceContext.py") ) return 0
def testRecordWithNestedUnion(self): with patch('avro_to_python.utils.avro.types.array._reference_type') as patched_func: filepath = self.directory + '/RecordWithNestedUnion.avsc' reader = AvscReader(file=filepath) reader.read() correct_field = {'name': 'CommonReference', 'type': 'records.nested'} correct_references = [] patched_func.assert_called_with(field=correct_field, references=correct_references)
def setUp(self): """ place empty avsc files in /tmp dir for testing paths """ self.source = os.path.abspath(avro_to_python.__file__) \ .replace(get_joined_path('avro_to_python', '__init__.py'), 'tests/avsc/records') reader = AvscReader(directory=self.source) reader.read() writer = AvroWriter(reader.file_tree) self.write_path = os.path.abspath( self.source.replace(get_joined_path('tests', 'avsc', 'records'), '')) writer.write(root_dir=self.write_path) sys.path.append(self.write_path)
def testLogicalRecord(self): filepath = self.directory + '/RecordWithLogicalTypes.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test logical type was mapped to primitive self.assertEqual( obj.children['records'].files['RecordWithLogicalTypes'].fields['timestamp'].fieldtype, # NOQA 'primitive' )
def main(source, target, pip=None, author=None, package_version=None): """avro-to-python: compile avro avsc schemata to python classes """ if os.path.isfile(source): reader = AvscReader(file=source) else: reader = AvscReader(directory=source) reader.read() writer = AvroWriter(reader.file_tree, pip=pip, author=author, package_version=package_version) writer.write(root_dir=target) del reader return 0
def testReadRecordWithEnum(self): filepath = self.directory + '/RecordWithEnum.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test the enum was created flavor_dict = { 'name': 'Flavor', 'avrotype': 'enum', 'namespace': 'records.nested', 'schema': { 'type': 'enum', 'name': 'Flavor', 'namespace': 'records.nested', 'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'] }, 'imports': [], 'fields': {}, 'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'], 'default': None } self.assertEqual( obj.children['records'].children['nested'].files['Flavor'].__dict__, # NOQA flavor_dict ) # test the record referencing enum was created self.assertEqual( 'RecordWithEnum', obj.children['records'].files['RecordWithEnum'].name ) # test the RecordWithEnum imports are correct self.assertEqual( {'name': 'Flavor', 'namespace': 'records.nested'}, obj.children['records'].files['RecordWithEnum'].imports[0].__dict__ ) # test Enum has good symbols self.assertEqual( ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'], obj.children['records'].children['nested'].files['Flavor'].symbols )
def testUnionRecord(self): filepath = self.directory + '/RecordWithUnion.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test both union types have 2 possible fields self.assertEqual( len(obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types), # NOQA len(obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types) # NOQA ) # test the unions are correct type self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[0].avrotype, # NOQA 'string' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[1].avrotype, # NOQA 'null' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[0].avrotype, # NOQA 'int' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[1].fieldtype, # NOQA 'reference' ) # make sure the import is valid self.assertEqual( obj.children['records'].files['RecordWithUnion'].imports[0].name, 'Thing' )
def testReadRecordWithRecord(self): filepath = self.directory + '/RecordWithRecord.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree self.assertEqual( obj.name, '' ) self.assertEqual( list(obj.children.keys()), ['records'] ) self.assertEqual( obj.children['records'].files['Thing'].name, 'Thing' ) self.assertEqual( len(obj.children['records'].files['Thing'].fields), 1 ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].name, 'id' ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].fieldtype, 'primitive' ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].avrotype, 'int' ) # Record with Record has 2 fields self.assertEqual( len(obj.children['records'].files['RecordWithRecord'].fields), 2 ) # Should have 1 import self.assertEqual( len(obj.children['records'].files['RecordWithRecord'].imports), 1 ) # import should have correct namespace self.assertEqual( {'name': 'Thing', 'namespace': 'records'}, obj.children['records'].files['RecordWithRecord'].imports[0].__dict__ # NOQA ) # make sure thing2 field has a default self.assertEqual( obj.children['records'].files['RecordWithRecord'].fields['thing2'].default, # NOQA {'id': 0} ) # make sure thing1 field has no default self.assertEqual( obj.children['records'].files['RecordWithRecord'].fields['thing1'].default, # NOQA None )