def test_get_system_file_path(self): """ tests that you can get the system path of a file """ expected = get_joined_path(self.nest_dir, 'test2.avsc') # change dir to alter local path cwd = os.getcwd() os.chdir(self.root) tmp_wd = os.getcwd() expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc') path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc')) # change dir to alter local path cwd = os.getcwd() os.chdir(self.nest_dir) print(os.getcwd()) path2 = get_system_path('test2.avsc') # change back to original dir to prevent errors os.chdir(cwd) self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}') self.assertEqual(expected, path2, f'paths not equal: {expected} != {path2}')
def test_is_avsc_file(self): """ test the is_avsc_file function """ is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc')) is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt')) self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!') self.assertFalse(is_not_avsc, '/tmp/avro/nest/bad.txt is an avsc file!')
def test_verify_path_exists(self): """ tests that file exists function works """ should_exist = verify_path_exists( get_joined_path(self.nest_dir, 'test2.avsc')) should_not_exist = verify_path_exists( get_joined_path(self.temp_dir, 'test2.avsc')) self.assertTrue(should_exist, 'expected /tmp/avro/nest/test2.avsc to exist') self.assertFalse(should_not_exist, 'expected /tmp/avro/test2.avsc to not exist')
def setUp(self): """ place empty avsc files in /tmp dir for testing paths """ self.source = os.path.abspath(avro_to_python.__file__) \ .replace(get_joined_path('avro_to_python', '__init__.py'), 'tests/avsc/records') reader = AvscReader(directory=self.source) reader.read() writer = AvroWriter(reader.file_tree) self.write_path = os.path.abspath( self.source.replace(get_joined_path('tests', 'avsc', 'records'), '')) writer.write(root_dir=self.write_path) sys.path.append(self.write_path)
def setUp(self): """ place empty avsc files in /tmp dir for testing paths """ # create tmp dir and files for path testing os.makedirs(self.temp_dir) os.makedirs(self.nest_dir) with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f: f.write('this is a test file') with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f: f.write('this is another test file') with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f: f.write('this is not an avsc file')
def test_get_avsc_files(self): """ tests that avsc files are read correctly """ directory = self.temp_dir root = os.path.dirname(directory) base_path = os.path.abspath(get_joined_path(root, directory)) expected = [ get_joined_path(base_path, 'test1.avsc'), get_joined_path(base_path, 'nest', 'test2.avsc') ] files = get_avsc_files(directory) bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files self.assertEqual(expected, files, 'expected values are not the same') self.assertTrue(bad_file, 'bad.txt was found in the files!!')
def test_verify_or_create_namespace_path(self): """ tests that verify_or_create_namespace_path works """ rootdir = self.root namespace = 'test.namespace.path' verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace) self.assertTrue( verify_path_exists( get_joined_path(self.test_dir, 'namespace', 'path')), 'should have created the path from a namespace') verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace) self.assertTrue( verify_path_exists( get_joined_path(self.test_dir, 'namespace', 'path')), 'should have created the path from a namespace') shutil.rmtree(self.test_dir)
class AvroReaderTests(unittest.TestCase): directory = os.path.abspath(avro_to_python.__file__) \ .replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records') def setUp(self): pass def tearDown(self): pass def testReadThing(self): filepath = self.directory + '/Thing.avsc' reader = AvscReader(file=filepath) reader.read() # test node tree structure is correct obj = reader.file_tree self.assertEqual( obj.name, '' ) self.assertEqual( list(obj.children.keys()), ['records'] ) # test Thing file is in correct place self.assertEqual( obj.children['records'].files['Thing'].name, 'Thing' ) # test Thing.id exists and only once self.assertEqual( len(obj.children['records'].files['Thing'].fields), 1 ) # test thing.id field has necessary information self.assertEqual( obj.children['records'].files['Thing'].fields['id'].avrotype, 'int' ) def testReadRecordWithRecord(self): filepath = self.directory + '/RecordWithRecord.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree self.assertEqual( obj.name, '' ) self.assertEqual( list(obj.children.keys()), ['records'] ) self.assertEqual( obj.children['records'].files['Thing'].name, 'Thing' ) self.assertEqual( len(obj.children['records'].files['Thing'].fields), 1 ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].name, 'id' ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].fieldtype, 'primitive' ) self.assertEqual( obj.children['records'].files['Thing'].fields['id'].avrotype, 'int' ) # Record with Record has 2 fields self.assertEqual( len(obj.children['records'].files['RecordWithRecord'].fields), 2 ) # Should have 1 import self.assertEqual( len(obj.children['records'].files['RecordWithRecord'].imports), 1 ) # import should have correct namespace self.assertEqual( {'name': 'Thing', 'namespace': 'records'}, obj.children['records'].files['RecordWithRecord'].imports[0].__dict__ # NOQA ) # make sure thing2 field has a default self.assertEqual( obj.children['records'].files['RecordWithRecord'].fields['thing2'].default, # NOQA {'id': 0} ) # make sure thing1 field has no default self.assertEqual( obj.children['records'].files['RecordWithRecord'].fields['thing1'].default, # NOQA None ) def testReadRecordWithEnum(self): filepath = self.directory + '/RecordWithEnum.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test the enum was created flavor_dict = { 'name': 'Flavor', 'avrotype': 'enum', 'namespace': 'records.nested', 'schema': { 'type': 'enum', 'name': 'Flavor', 'namespace': 'records.nested', 'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'] }, 'imports': [], 'fields': {}, 'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'], 'default': None } self.assertEqual( obj.children['records'].children['nested'].files['Flavor'].__dict__, # NOQA flavor_dict ) # test the record referencing enum was created self.assertEqual( 'RecordWithEnum', obj.children['records'].files['RecordWithEnum'].name ) # test the RecordWithEnum imports are correct self.assertEqual( {'name': 'Flavor', 'namespace': 'records.nested'}, obj.children['records'].files['RecordWithEnum'].imports[0].__dict__ ) # test Enum has good symbols self.assertEqual( ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'], obj.children['records'].children['nested'].files['Flavor'].symbols ) def testLogicalRecord(self): filepath = self.directory + '/RecordWithLogicalTypes.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test logical type was mapped to primitive self.assertEqual( obj.children['records'].files['RecordWithLogicalTypes'].fields['timestamp'].fieldtype, # NOQA 'primitive' ) def testUnionRecord(self): filepath = self.directory + '/RecordWithUnion.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # test both union types have 2 possible fields self.assertEqual( len(obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types), # NOQA len(obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types) # NOQA ) # test the unions are correct type self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[0].avrotype, # NOQA 'string' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[1].avrotype, # NOQA 'null' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[0].avrotype, # NOQA 'int' ) self.assertEqual( obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[1].fieldtype, # NOQA 'reference' ) # make sure the import is valid self.assertEqual( obj.children['records'].files['RecordWithUnion'].imports[0].name, 'Thing' ) def testRecordWithArray(self): filepath = self.directory + '/RecordWithArray.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # should have 3 fields self.assertEqual( len(obj.children['records'].files['RecordWithArray'].fields), 3 ) # field 0 should be of type reference self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['things'].array_item_type.fieldtype, # NOQA 'reference' ) # field 1 should be of type primitive self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['numbers'].array_item_type.fieldtype, # NOQA 'primitive' ) # field 2 should be of type reference self.assertEqual( obj.children['records'].files['RecordWithArray'].fields['things2'].array_item_type.fieldtype, # NOQA 'reference' ) def testRecordWithMap(self): filepath = self.directory + '/RecordWithMap.avsc' reader = AvscReader(file=filepath) reader.read() # TestReadThing tests from earlier since Thing is same schema obj = reader.file_tree # should have 1 field self.assertEqual( len(obj.children['records'].files['RecordWithMap'].fields), 4 ) # field should be of type map self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['thingMap'].fieldtype, # NOQA 'map' ) # thingmap field should be of type Thing self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['thingMap'].map_type.reference_name, # NOQA 'Thing' ) # map int field should be of type Thing self.assertEqual( obj.children['records'].files['RecordWithMap'].fields['intMap'].map_type.avrotype, # NOQA 'int' ) def testRecordWithNestedMap(self): filepath = self.directory + '/RecordWithNestedMap.avsc' reader = AvscReader(file=filepath) reader.read() # test parsing works for nested maps obj = reader.file_tree file = obj.children['records'].files['RecordWithNestedMap'] self.assertEqual( len(obj.children['records'].files['RecordWithNestedMap'].fields), 4 ) # assert reader picked up the nested map self.assertEqual( file.fields['nestedThingMap'].map_type.map_type.name, 'Thing' ) self.assertEqual( file.fields['nestedThingMap'].map_type.map_type.fieldtype, 'reference' ) # assert reader picked up nested array in map self.assertEqual( file.fields['mappedThingArray'].map_type.array_item_type.name, 'Thing' ) self.assertEqual( file.fields['mappedThingArray'].map_type.array_item_type.fieldtype, 'reference' ) def testRecordWithNestedUnion(self): with patch('avro_to_python.utils.avro.types.array._reference_type') as patched_func: filepath = self.directory + '/RecordWithNestedUnion.avsc' reader = AvscReader(file=filepath) reader.read() correct_field = {'name': 'CommonReference', 'type': 'records.nested'} correct_references = [] patched_func.assert_called_with(field=correct_field, references=correct_references)
""" Writer class for writing python avro files """ import json from jinja2 import Environment, FileSystemLoader from avro_to_python.classes.node import Node from avro_to_python.utils.avro.helpers import get_union_types from avro_to_python.utils.avro.primitive_types import PRIMITIVE_TYPE_MAP from avro_to_python.utils.paths import (get_system_path, verify_or_create_namespace_path, get_or_create_path, get_joined_path) TEMPLATE_PATH = __file__.replace(get_joined_path('writer', 'writer.py'), 'templates/') TEMPLATE_PATH = get_system_path(TEMPLATE_PATH) class AvroWriter(object): """ writer class for writing python files Should initiate around a tree object with nodes as: { 'children': {}, 'files': {}, 'visited': False } The "keys" of the children are the namespace names along avro namespace paths. The Files are the actual files within the
def setUp(self): """ place empty avsc files in /tmp dir for testing paths """ self.source = os.path.abspath(avro_to_python.__file__) \ .replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records')
class PathTests(unittest.TestCase): root = os.path.abspath("/tmp") temp_dir = os.path.abspath(get_joined_path(root, 'avro')) nest_dir = get_joined_path(temp_dir, 'nest') test_dir = get_joined_path(root, 'test') def setUp(self): """ place empty avsc files in /tmp dir for testing paths """ # create tmp dir and files for path testing os.makedirs(self.temp_dir) os.makedirs(self.nest_dir) with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f: f.write('this is a test file') with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f: f.write('this is another test file') with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f: f.write('this is not an avsc file') def tearDown(self): # remove the files we created shutil.rmtree(self.temp_dir) def test_get_avsc_files(self): """ tests that avsc files are read correctly """ directory = self.temp_dir root = os.path.dirname(directory) base_path = os.path.abspath(get_joined_path(root, directory)) expected = [ get_joined_path(base_path, 'test1.avsc'), get_joined_path(base_path, 'nest', 'test2.avsc') ] files = get_avsc_files(directory) bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files self.assertEqual(expected, files, 'expected values are not the same') self.assertTrue(bad_file, 'bad.txt was found in the files!!') def test_verify_path_exists(self): """ tests that file exists function works """ should_exist = verify_path_exists( get_joined_path(self.nest_dir, 'test2.avsc')) should_not_exist = verify_path_exists( get_joined_path(self.temp_dir, 'test2.avsc')) self.assertTrue(should_exist, 'expected /tmp/avro/nest/test2.avsc to exist') self.assertFalse(should_not_exist, 'expected /tmp/avro/test2.avsc to not exist') def test_is_avsc_file(self): """ test the is_avsc_file function """ is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc')) is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt')) self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!') self.assertFalse(is_not_avsc, '/tmp/avro/nest/bad.txt is an avsc file!') def test_get_system_file_path(self): """ tests that you can get the system path of a file """ expected = get_joined_path(self.nest_dir, 'test2.avsc') # change dir to alter local path cwd = os.getcwd() os.chdir(self.root) tmp_wd = os.getcwd() expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc') path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc')) # change dir to alter local path cwd = os.getcwd() os.chdir(self.nest_dir) print(os.getcwd()) path2 = get_system_path('test2.avsc') # change back to original dir to prevent errors os.chdir(cwd) self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}') self.assertEqual(expected, path2, f'paths not equal: {expected} != {path2}') def test_verify_or_create_namespace_path(self): """ tests that verify_or_create_namespace_path works """ rootdir = self.root namespace = 'test.namespace.path' verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace) self.assertTrue( verify_path_exists( get_joined_path(self.test_dir, 'namespace', 'path')), 'should have created the path from a namespace') verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace) self.assertTrue( verify_path_exists( get_joined_path(self.test_dir, 'namespace', 'path')), 'should have created the path from a namespace') shutil.rmtree(self.test_dir) def test_get_or_create_path(self): """ tests that get_or_create_path works """ path = self.test_dir get_or_create_path(path=path) self.assertTrue(verify_path_exists(self.test_dir), 'should have created path at /tmp/test') os.rmdir(path) os.chdir(self.root) get_or_create_path(path='test') self.assertTrue( verify_path_exists(self.test_dir), 'should have created path at /tmp/test from relative path') shutil.rmtree(path)