def test_get_system_file_path(self):
""" tests that you can get the system path of a file """
expected = get_joined_path(self.nest_dir, 'test2.avsc')
# change dir to alter local path
cwd = os.getcwd()
os.chdir(self.root)
tmp_wd = os.getcwd()
expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc')
path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc'))
# change dir to alter local path
cwd = os.getcwd()
os.chdir(self.nest_dir)
print(os.getcwd())
path2 = get_system_path('test2.avsc')
# change back to original dir to prevent errors
os.chdir(cwd)
self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}')
self.assertEqual(expected, path2,
f'paths not equal: {expected} != {path2}')
def test_is_avsc_file(self):
""" test the is_avsc_file function """
is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc'))
is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt'))
self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!')
self.assertFalse(is_not_avsc,
'/tmp/avro/nest/bad.txt is an avsc file!')
def test_verify_path_exists(self):
""" tests that file exists function works """
should_exist = verify_path_exists(
get_joined_path(self.nest_dir, 'test2.avsc'))
should_not_exist = verify_path_exists(
get_joined_path(self.temp_dir, 'test2.avsc'))
self.assertTrue(should_exist,
'expected /tmp/avro/nest/test2.avsc to exist')
self.assertFalse(should_not_exist,
'expected /tmp/avro/test2.avsc to not exist')
def setUp(self):
""" place empty avsc files in /tmp dir for testing paths """
self.source = os.path.abspath(avro_to_python.__file__) \
.replace(get_joined_path('avro_to_python', '__init__.py'), 'tests/avsc/records')
reader = AvscReader(directory=self.source)
reader.read()
writer = AvroWriter(reader.file_tree)
self.write_path = os.path.abspath(
self.source.replace(get_joined_path('tests', 'avsc', 'records'),
''))
writer.write(root_dir=self.write_path)
sys.path.append(self.write_path)
def setUp(self):
""" place empty avsc files in /tmp dir for testing paths """
# create tmp dir and files for path testing
os.makedirs(self.temp_dir)
os.makedirs(self.nest_dir)
with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f:
f.write('this is a test file')
with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f:
f.write('this is another test file')
with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f:
f.write('this is not an avsc file')
def test_get_avsc_files(self):
""" tests that avsc files are read correctly """
directory = self.temp_dir
root = os.path.dirname(directory)
base_path = os.path.abspath(get_joined_path(root, directory))
expected = [
get_joined_path(base_path, 'test1.avsc'),
get_joined_path(base_path, 'nest', 'test2.avsc')
]
files = get_avsc_files(directory)
bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files
self.assertEqual(expected, files, 'expected values are not the same')
self.assertTrue(bad_file, 'bad.txt was found in the files!!')
def test_verify_or_create_namespace_path(self):
""" tests that verify_or_create_namespace_path works """
rootdir = self.root
namespace = 'test.namespace.path'
verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)
self.assertTrue(
verify_path_exists(
get_joined_path(self.test_dir, 'namespace', 'path')),
'should have created the path from a namespace')
verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)
self.assertTrue(
verify_path_exists(
get_joined_path(self.test_dir, 'namespace', 'path')),
'should have created the path from a namespace')
shutil.rmtree(self.test_dir)
class AvroReaderTests(unittest.TestCase):
directory = os.path.abspath(avro_to_python.__file__) \
.replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records')
def setUp(self):
pass
def tearDown(self):
pass
def testReadThing(self):
filepath = self.directory + '/Thing.avsc'
reader = AvscReader(file=filepath)
reader.read()
# test node tree structure is correct
obj = reader.file_tree
self.assertEqual(
obj.name,
''
)
self.assertEqual(
list(obj.children.keys()),
['records']
)
# test Thing file is in correct place
self.assertEqual(
obj.children['records'].files['Thing'].name,
'Thing'
)
# test Thing.id exists and only once
self.assertEqual(
len(obj.children['records'].files['Thing'].fields),
1
)
# test thing.id field has necessary information
self.assertEqual(
obj.children['records'].files['Thing'].fields['id'].avrotype,
'int'
)
def testReadRecordWithRecord(self):
filepath = self.directory + '/RecordWithRecord.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
self.assertEqual(
obj.name,
''
)
self.assertEqual(
list(obj.children.keys()),
['records']
)
self.assertEqual(
obj.children['records'].files['Thing'].name,
'Thing'
)
self.assertEqual(
len(obj.children['records'].files['Thing'].fields),
1
)
self.assertEqual(
obj.children['records'].files['Thing'].fields['id'].name,
'id'
)
self.assertEqual(
obj.children['records'].files['Thing'].fields['id'].fieldtype,
'primitive'
)
self.assertEqual(
obj.children['records'].files['Thing'].fields['id'].avrotype,
'int'
)
# Record with Record has 2 fields
self.assertEqual(
len(obj.children['records'].files['RecordWithRecord'].fields),
2
)
# Should have 1 import
self.assertEqual(
len(obj.children['records'].files['RecordWithRecord'].imports),
1
)
# import should have correct namespace
self.assertEqual(
{'name': 'Thing', 'namespace': 'records'},
obj.children['records'].files['RecordWithRecord'].imports[0].__dict__ # NOQA
)
# make sure thing2 field has a default
self.assertEqual(
obj.children['records'].files['RecordWithRecord'].fields['thing2'].default, # NOQA
{'id': 0}
)
# make sure thing1 field has no default
self.assertEqual(
obj.children['records'].files['RecordWithRecord'].fields['thing1'].default, # NOQA
None
)
def testReadRecordWithEnum(self):
filepath = self.directory + '/RecordWithEnum.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
# test the enum was created
flavor_dict = {
'name': 'Flavor',
'avrotype': 'enum',
'namespace': 'records.nested',
'schema': {
'type': 'enum',
'name': 'Flavor',
'namespace': 'records.nested',
'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY']
},
'imports': [],
'fields': {},
'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'],
'default': None
}
self.assertEqual(
obj.children['records'].children['nested'].files['Flavor'].__dict__, # NOQA
flavor_dict
)
# test the record referencing enum was created
self.assertEqual(
'RecordWithEnum',
obj.children['records'].files['RecordWithEnum'].name
)
# test the RecordWithEnum imports are correct
self.assertEqual(
{'name': 'Flavor', 'namespace': 'records.nested'},
obj.children['records'].files['RecordWithEnum'].imports[0].__dict__
)
# test Enum has good symbols
self.assertEqual(
['VANILLA', 'CHOCOLATE', 'STRAWBERRY'],
obj.children['records'].children['nested'].files['Flavor'].symbols
)
def testLogicalRecord(self):
filepath = self.directory + '/RecordWithLogicalTypes.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
# test logical type was mapped to primitive
self.assertEqual(
obj.children['records'].files['RecordWithLogicalTypes'].fields['timestamp'].fieldtype, # NOQA
'primitive'
)
def testUnionRecord(self):
filepath = self.directory + '/RecordWithUnion.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
# test both union types have 2 possible fields
self.assertEqual(
len(obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types), # NOQA
len(obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types) # NOQA
)
# test the unions are correct type
self.assertEqual(
obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[0].avrotype, # NOQA
'string'
)
self.assertEqual(
obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[1].avrotype, # NOQA
'null'
)
self.assertEqual(
obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[0].avrotype, # NOQA
'int'
)
self.assertEqual(
obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[1].fieldtype, # NOQA
'reference'
)
# make sure the import is valid
self.assertEqual(
obj.children['records'].files['RecordWithUnion'].imports[0].name,
'Thing'
)
def testRecordWithArray(self):
filepath = self.directory + '/RecordWithArray.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
# should have 3 fields
self.assertEqual(
len(obj.children['records'].files['RecordWithArray'].fields),
3
)
# field 0 should be of type reference
self.assertEqual(
obj.children['records'].files['RecordWithArray'].fields['things'].array_item_type.fieldtype, # NOQA
'reference'
)
# field 1 should be of type primitive
self.assertEqual(
obj.children['records'].files['RecordWithArray'].fields['numbers'].array_item_type.fieldtype, # NOQA
'primitive'
)
# field 2 should be of type reference
self.assertEqual(
obj.children['records'].files['RecordWithArray'].fields['things2'].array_item_type.fieldtype, # NOQA
'reference'
)
def testRecordWithMap(self):
filepath = self.directory + '/RecordWithMap.avsc'
reader = AvscReader(file=filepath)
reader.read()
# TestReadThing tests from earlier since Thing is same schema
obj = reader.file_tree
# should have 1 field
self.assertEqual(
len(obj.children['records'].files['RecordWithMap'].fields),
4
)
# field should be of type map
self.assertEqual(
obj.children['records'].files['RecordWithMap'].fields['thingMap'].fieldtype, # NOQA
'map'
)
# thingmap field should be of type Thing
self.assertEqual(
obj.children['records'].files['RecordWithMap'].fields['thingMap'].map_type.reference_name, # NOQA
'Thing'
)
# map int field should be of type Thing
self.assertEqual(
obj.children['records'].files['RecordWithMap'].fields['intMap'].map_type.avrotype, # NOQA
'int'
)
def testRecordWithNestedMap(self):
filepath = self.directory + '/RecordWithNestedMap.avsc'
reader = AvscReader(file=filepath)
reader.read()
# test parsing works for nested maps
obj = reader.file_tree
file = obj.children['records'].files['RecordWithNestedMap']
self.assertEqual(
len(obj.children['records'].files['RecordWithNestedMap'].fields),
4
)
# assert reader picked up the nested map
self.assertEqual(
file.fields['nestedThingMap'].map_type.map_type.name,
'Thing'
)
self.assertEqual(
file.fields['nestedThingMap'].map_type.map_type.fieldtype,
'reference'
)
# assert reader picked up nested array in map
self.assertEqual(
file.fields['mappedThingArray'].map_type.array_item_type.name,
'Thing'
)
self.assertEqual(
file.fields['mappedThingArray'].map_type.array_item_type.fieldtype,
'reference'
)
def testRecordWithNestedUnion(self):
with patch('avro_to_python.utils.avro.types.array._reference_type') as patched_func:
filepath = self.directory + '/RecordWithNestedUnion.avsc'
reader = AvscReader(file=filepath)
reader.read()
correct_field = {'name': 'CommonReference', 'type': 'records.nested'}
correct_references = []
patched_func.assert_called_with(field=correct_field, references=correct_references)
""" Writer class for writing python avro files """
import json
from jinja2 import Environment, FileSystemLoader
from avro_to_python.classes.node import Node
from avro_to_python.utils.avro.helpers import get_union_types
from avro_to_python.utils.avro.primitive_types import PRIMITIVE_TYPE_MAP
from avro_to_python.utils.paths import (get_system_path,
verify_or_create_namespace_path,
get_or_create_path, get_joined_path)
TEMPLATE_PATH = __file__.replace(get_joined_path('writer', 'writer.py'),
'templates/')
TEMPLATE_PATH = get_system_path(TEMPLATE_PATH)
class AvroWriter(object):
""" writer class for writing python files
Should initiate around a tree object with nodes as:
{
'children': {},
'files': {},
'visited': False
}
The "keys" of the children are the namespace names along avro
namespace paths. The Files are the actual files within the
def setUp(self):
""" place empty avsc files in /tmp dir for testing paths """
self.source = os.path.abspath(avro_to_python.__file__) \
.replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records')
class PathTests(unittest.TestCase):
root = os.path.abspath("/tmp")
temp_dir = os.path.abspath(get_joined_path(root, 'avro'))
nest_dir = get_joined_path(temp_dir, 'nest')
test_dir = get_joined_path(root, 'test')
def setUp(self):
""" place empty avsc files in /tmp dir for testing paths """
# create tmp dir and files for path testing
os.makedirs(self.temp_dir)
os.makedirs(self.nest_dir)
with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f:
f.write('this is a test file')
with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f:
f.write('this is another test file')
with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f:
f.write('this is not an avsc file')
def tearDown(self):
# remove the files we created
shutil.rmtree(self.temp_dir)
def test_get_avsc_files(self):
""" tests that avsc files are read correctly """
directory = self.temp_dir
root = os.path.dirname(directory)
base_path = os.path.abspath(get_joined_path(root, directory))
expected = [
get_joined_path(base_path, 'test1.avsc'),
get_joined_path(base_path, 'nest', 'test2.avsc')
]
files = get_avsc_files(directory)
bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files
self.assertEqual(expected, files, 'expected values are not the same')
self.assertTrue(bad_file, 'bad.txt was found in the files!!')
def test_verify_path_exists(self):
""" tests that file exists function works """
should_exist = verify_path_exists(
get_joined_path(self.nest_dir, 'test2.avsc'))
should_not_exist = verify_path_exists(
get_joined_path(self.temp_dir, 'test2.avsc'))
self.assertTrue(should_exist,
'expected /tmp/avro/nest/test2.avsc to exist')
self.assertFalse(should_not_exist,
'expected /tmp/avro/test2.avsc to not exist')
def test_is_avsc_file(self):
""" test the is_avsc_file function """
is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc'))
is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt'))
self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!')
self.assertFalse(is_not_avsc,
'/tmp/avro/nest/bad.txt is an avsc file!')
def test_get_system_file_path(self):
""" tests that you can get the system path of a file """
expected = get_joined_path(self.nest_dir, 'test2.avsc')
# change dir to alter local path
cwd = os.getcwd()
os.chdir(self.root)
tmp_wd = os.getcwd()
expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc')
path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc'))
# change dir to alter local path
cwd = os.getcwd()
os.chdir(self.nest_dir)
print(os.getcwd())
path2 = get_system_path('test2.avsc')
# change back to original dir to prevent errors
os.chdir(cwd)
self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}')
self.assertEqual(expected, path2,
f'paths not equal: {expected} != {path2}')
def test_verify_or_create_namespace_path(self):
""" tests that verify_or_create_namespace_path works """
rootdir = self.root
namespace = 'test.namespace.path'
verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)
self.assertTrue(
verify_path_exists(
get_joined_path(self.test_dir, 'namespace', 'path')),
'should have created the path from a namespace')
verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)
self.assertTrue(
verify_path_exists(
get_joined_path(self.test_dir, 'namespace', 'path')),
'should have created the path from a namespace')
shutil.rmtree(self.test_dir)
def test_get_or_create_path(self):
""" tests that get_or_create_path works """
path = self.test_dir
get_or_create_path(path=path)
self.assertTrue(verify_path_exists(self.test_dir),
'should have created path at /tmp/test')
os.rmdir(path)
os.chdir(self.root)
get_or_create_path(path='test')
self.assertTrue(
verify_path_exists(self.test_dir),
'should have created path at /tmp/test from relative path')
shutil.rmtree(path)