Beispiel #1
0
    def test_get_system_file_path(self):
        """ tests that you can get the system path of a file """
        expected = get_joined_path(self.nest_dir, 'test2.avsc')

        # change dir to alter local path
        cwd = os.getcwd()
        os.chdir(self.root)

        tmp_wd = os.getcwd()

        expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc')

        path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc'))

        # change dir to alter local path
        cwd = os.getcwd()
        os.chdir(self.nest_dir)
        print(os.getcwd())

        path2 = get_system_path('test2.avsc')

        # change back to original dir to prevent errors
        os.chdir(cwd)

        self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}')

        self.assertEqual(expected, path2,
                         f'paths not equal: {expected} != {path2}')
Beispiel #2
0
    def test_is_avsc_file(self):
        """ test the is_avsc_file function """
        is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc'))
        is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt'))

        self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!')
        self.assertFalse(is_not_avsc,
                         '/tmp/avro/nest/bad.txt is an avsc file!')
Beispiel #3
0
 def test_verify_path_exists(self):
     """ tests that file exists function works """
     should_exist = verify_path_exists(
         get_joined_path(self.nest_dir, 'test2.avsc'))
     should_not_exist = verify_path_exists(
         get_joined_path(self.temp_dir, 'test2.avsc'))
     self.assertTrue(should_exist,
                     'expected /tmp/avro/nest/test2.avsc to exist')
     self.assertFalse(should_not_exist,
                      'expected /tmp/avro/test2.avsc to not exist')
Beispiel #4
0
    def setUp(self):
        """ place empty avsc files in /tmp dir for testing paths """
        self.source = os.path.abspath(avro_to_python.__file__) \
            .replace(get_joined_path('avro_to_python', '__init__.py'), 'tests/avsc/records')
        reader = AvscReader(directory=self.source)
        reader.read()
        writer = AvroWriter(reader.file_tree)

        self.write_path = os.path.abspath(
            self.source.replace(get_joined_path('tests', 'avsc', 'records'),
                                ''))

        writer.write(root_dir=self.write_path)
        sys.path.append(self.write_path)
Beispiel #5
0
    def setUp(self):
        """ place empty avsc files in /tmp dir for testing paths """

        # create tmp dir and files for path testing

        os.makedirs(self.temp_dir)
        os.makedirs(self.nest_dir)

        with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f:
            f.write('this is a test file')

        with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f:
            f.write('this is another test file')

        with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f:
            f.write('this is not an avsc file')
Beispiel #6
0
    def test_get_avsc_files(self):
        """ tests that avsc files are read correctly """
        directory = self.temp_dir
        root = os.path.dirname(directory)
        base_path = os.path.abspath(get_joined_path(root, directory))
        expected = [
            get_joined_path(base_path, 'test1.avsc'),
            get_joined_path(base_path, 'nest', 'test2.avsc')
        ]

        files = get_avsc_files(directory)

        bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files

        self.assertEqual(expected, files, 'expected values are not the same')

        self.assertTrue(bad_file, 'bad.txt was found in the files!!')
Beispiel #7
0
    def test_verify_or_create_namespace_path(self):
        """ tests that verify_or_create_namespace_path works """
        rootdir = self.root
        namespace = 'test.namespace.path'

        verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)

        self.assertTrue(
            verify_path_exists(
                get_joined_path(self.test_dir, 'namespace', 'path')),
            'should have created the path from a namespace')

        verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)

        self.assertTrue(
            verify_path_exists(
                get_joined_path(self.test_dir, 'namespace', 'path')),
            'should have created the path from a namespace')
        shutil.rmtree(self.test_dir)
Beispiel #8
0
class AvroReaderTests(unittest.TestCase):

    directory = os.path.abspath(avro_to_python.__file__) \
        .replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records')

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def testReadThing(self):
        filepath = self.directory + '/Thing.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # test node tree structure is correct
        obj = reader.file_tree
        self.assertEqual(
            obj.name,
            ''
        )

        self.assertEqual(
            list(obj.children.keys()),
            ['records']
        )

        # test Thing file is in correct place
        self.assertEqual(
            obj.children['records'].files['Thing'].name,
            'Thing'
        )

        # test Thing.id exists and only once
        self.assertEqual(
            len(obj.children['records'].files['Thing'].fields),
            1
        )

        # test thing.id field has necessary information
        self.assertEqual(
            obj.children['records'].files['Thing'].fields['id'].avrotype,
            'int'
        )

    def testReadRecordWithRecord(self):
        filepath = self.directory + '/RecordWithRecord.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree
        self.assertEqual(
            obj.name,
            ''
        )

        self.assertEqual(
            list(obj.children.keys()),
            ['records']
        )

        self.assertEqual(
            obj.children['records'].files['Thing'].name,
            'Thing'
        )

        self.assertEqual(
            len(obj.children['records'].files['Thing'].fields),
            1
        )

        self.assertEqual(
            obj.children['records'].files['Thing'].fields['id'].name,
            'id'
        )

        self.assertEqual(
            obj.children['records'].files['Thing'].fields['id'].fieldtype,
            'primitive'
        )

        self.assertEqual(
            obj.children['records'].files['Thing'].fields['id'].avrotype,
            'int'
        )

        # Record with Record has 2 fields
        self.assertEqual(
            len(obj.children['records'].files['RecordWithRecord'].fields),
            2
        )

        # Should have 1 import
        self.assertEqual(
            len(obj.children['records'].files['RecordWithRecord'].imports),
            1
        )

        # import should have correct namespace
        self.assertEqual(
            {'name': 'Thing', 'namespace': 'records'},
            obj.children['records'].files['RecordWithRecord'].imports[0].__dict__  # NOQA
        )

        # make sure thing2 field has a default
        self.assertEqual(
            obj.children['records'].files['RecordWithRecord'].fields['thing2'].default,  # NOQA
            {'id': 0}
        )

        # make sure thing1 field has no default
        self.assertEqual(
            obj.children['records'].files['RecordWithRecord'].fields['thing1'].default,  # NOQA
            None
        )

    def testReadRecordWithEnum(self):
        filepath = self.directory + '/RecordWithEnum.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree

        # test the enum was created
        flavor_dict = {
            'name': 'Flavor',
            'avrotype': 'enum',
            'namespace': 'records.nested',
            'schema': {
                'type': 'enum',
                'name': 'Flavor',
                'namespace': 'records.nested',
                'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY']
            },
            'imports': [],
            'fields': {},
            'symbols': ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'],
            'default': None
        }
        self.assertEqual(
            obj.children['records'].children['nested'].files['Flavor'].__dict__,  # NOQA
            flavor_dict
        )

        # test the record referencing enum was created
        self.assertEqual(
            'RecordWithEnum',
            obj.children['records'].files['RecordWithEnum'].name
        )

        # test the RecordWithEnum imports are correct
        self.assertEqual(
            {'name': 'Flavor', 'namespace': 'records.nested'},
            obj.children['records'].files['RecordWithEnum'].imports[0].__dict__
        )

        # test Enum has good symbols
        self.assertEqual(
            ['VANILLA', 'CHOCOLATE', 'STRAWBERRY'],
            obj.children['records'].children['nested'].files['Flavor'].symbols
        )

    def testLogicalRecord(self):
        filepath = self.directory + '/RecordWithLogicalTypes.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree

        # test logical type was mapped to primitive
        self.assertEqual(
            obj.children['records'].files['RecordWithLogicalTypes'].fields['timestamp'].fieldtype,  # NOQA
            'primitive'
        )

    def testUnionRecord(self):
        filepath = self.directory + '/RecordWithUnion.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree

        # test both union types have 2 possible fields
        self.assertEqual(
            len(obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types),  # NOQA
            len(obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types)  # NOQA
        )

        # test the unions are correct type
        self.assertEqual(
            obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[0].avrotype,  # NOQA
            'string'
        )

        self.assertEqual(
            obj.children['records'].files['RecordWithUnion'].fields['optionalString'].union_types[1].avrotype,  # NOQA
            'null'
        )

        self.assertEqual(
            obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[0].avrotype,  # NOQA
            'int'
        )

        self.assertEqual(
            obj.children['records'].files['RecordWithUnion'].fields['intOrThing'].union_types[1].fieldtype,  # NOQA
            'reference'
        )

        # make sure the import is valid
        self.assertEqual(
            obj.children['records'].files['RecordWithUnion'].imports[0].name,
            'Thing'
        )

    def testRecordWithArray(self):
        filepath = self.directory + '/RecordWithArray.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree

        # should have 3 fields
        self.assertEqual(
            len(obj.children['records'].files['RecordWithArray'].fields),
            3
        )

        # field 0 should be of type reference
        self.assertEqual(
            obj.children['records'].files['RecordWithArray'].fields['things'].array_item_type.fieldtype,  # NOQA
            'reference'
        )

        # field 1 should be of type primitive
        self.assertEqual(
            obj.children['records'].files['RecordWithArray'].fields['numbers'].array_item_type.fieldtype,  # NOQA
            'primitive'
        )

        # field 2 should be of type reference
        self.assertEqual(
            obj.children['records'].files['RecordWithArray'].fields['things2'].array_item_type.fieldtype,  # NOQA
            'reference'
        )

    def testRecordWithMap(self):
        filepath = self.directory + '/RecordWithMap.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # TestReadThing tests from earlier since Thing is same schema
        obj = reader.file_tree

        # should have 1 field
        self.assertEqual(
            len(obj.children['records'].files['RecordWithMap'].fields),
            4
        )

        # field should be of type map
        self.assertEqual(
            obj.children['records'].files['RecordWithMap'].fields['thingMap'].fieldtype,  # NOQA
            'map'
        )

        # thingmap field should be of type Thing
        self.assertEqual(
            obj.children['records'].files['RecordWithMap'].fields['thingMap'].map_type.reference_name,  # NOQA
            'Thing'
        )

        # map int field should be of type Thing
        self.assertEqual(
            obj.children['records'].files['RecordWithMap'].fields['intMap'].map_type.avrotype,  # NOQA
            'int'
        )

    def testRecordWithNestedMap(self):
        filepath = self.directory + '/RecordWithNestedMap.avsc'

        reader = AvscReader(file=filepath)
        reader.read()

        # test parsing works for nested maps
        obj = reader.file_tree
        file = obj.children['records'].files['RecordWithNestedMap']

        self.assertEqual(
            len(obj.children['records'].files['RecordWithNestedMap'].fields),
            4
        )

        # assert reader picked up the nested map
        self.assertEqual(
            file.fields['nestedThingMap'].map_type.map_type.name,
            'Thing'
        )
        self.assertEqual(
            file.fields['nestedThingMap'].map_type.map_type.fieldtype,
            'reference'
        )

        # assert reader picked up nested array in map
        self.assertEqual(
            file.fields['mappedThingArray'].map_type.array_item_type.name,
            'Thing'
        )

        self.assertEqual(
            file.fields['mappedThingArray'].map_type.array_item_type.fieldtype,
            'reference'
        )

    def testRecordWithNestedUnion(self):
        with patch('avro_to_python.utils.avro.types.array._reference_type') as patched_func:
            filepath = self.directory + '/RecordWithNestedUnion.avsc'
            reader = AvscReader(file=filepath)
            reader.read()

        correct_field = {'name': 'CommonReference', 'type': 'records.nested'}
        correct_references = []
        patched_func.assert_called_with(field=correct_field, references=correct_references)
Beispiel #9
0
""" Writer class for writing python avro files """

import json

from jinja2 import Environment, FileSystemLoader

from avro_to_python.classes.node import Node
from avro_to_python.utils.avro.helpers import get_union_types
from avro_to_python.utils.avro.primitive_types import PRIMITIVE_TYPE_MAP
from avro_to_python.utils.paths import (get_system_path,
                                        verify_or_create_namespace_path,
                                        get_or_create_path, get_joined_path)

TEMPLATE_PATH = __file__.replace(get_joined_path('writer', 'writer.py'),
                                 'templates/')
TEMPLATE_PATH = get_system_path(TEMPLATE_PATH)


class AvroWriter(object):
    """ writer class for writing python files

    Should initiate around a tree object with nodes as:

    {
        'children': {},
        'files': {},
        'visited': False
    }

    The "keys" of the children are the namespace names along avro
    namespace paths. The Files are the actual files within the
Beispiel #10
0
 def setUp(self):
     """ place empty avsc files in /tmp dir for testing paths """
     self.source = os.path.abspath(avro_to_python.__file__) \
         .replace(get_joined_path('avro_to_python','__init__.py'), 'tests/avsc/records')
Beispiel #11
0
class PathTests(unittest.TestCase):

    root = os.path.abspath("/tmp")
    temp_dir = os.path.abspath(get_joined_path(root, 'avro'))
    nest_dir = get_joined_path(temp_dir, 'nest')
    test_dir = get_joined_path(root, 'test')

    def setUp(self):
        """ place empty avsc files in /tmp dir for testing paths """

        # create tmp dir and files for path testing

        os.makedirs(self.temp_dir)
        os.makedirs(self.nest_dir)

        with open(get_joined_path(self.temp_dir, 'test1.avsc'), 'w') as f:
            f.write('this is a test file')

        with open(get_joined_path(self.nest_dir, 'test2.avsc'), 'w') as f:
            f.write('this is another test file')

        with open(get_joined_path(self.nest_dir, 'bad.txt'), 'w') as f:
            f.write('this is not an avsc file')

    def tearDown(self):
        # remove the files we created
        shutil.rmtree(self.temp_dir)

    def test_get_avsc_files(self):
        """ tests that avsc files are read correctly """
        directory = self.temp_dir
        root = os.path.dirname(directory)
        base_path = os.path.abspath(get_joined_path(root, directory))
        expected = [
            get_joined_path(base_path, 'test1.avsc'),
            get_joined_path(base_path, 'nest', 'test2.avsc')
        ]

        files = get_avsc_files(directory)

        bad_file = get_joined_path(self.nest_dir, 'bad.txt') not in files

        self.assertEqual(expected, files, 'expected values are not the same')

        self.assertTrue(bad_file, 'bad.txt was found in the files!!')

    def test_verify_path_exists(self):
        """ tests that file exists function works """
        should_exist = verify_path_exists(
            get_joined_path(self.nest_dir, 'test2.avsc'))
        should_not_exist = verify_path_exists(
            get_joined_path(self.temp_dir, 'test2.avsc'))
        self.assertTrue(should_exist,
                        'expected /tmp/avro/nest/test2.avsc to exist')
        self.assertFalse(should_not_exist,
                         'expected /tmp/avro/test2.avsc to not exist')

    def test_is_avsc_file(self):
        """ test the is_avsc_file function """
        is_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'test2.avsc'))
        is_not_avsc = is_avsc_file(get_joined_path(self.nest_dir, 'bad.txt'))

        self.assertTrue(is_avsc, '/tmp/avro/nest/test2.avsc is an avsc file!')
        self.assertFalse(is_not_avsc,
                         '/tmp/avro/nest/bad.txt is an avsc file!')

    def test_get_system_file_path(self):
        """ tests that you can get the system path of a file """
        expected = get_joined_path(self.nest_dir, 'test2.avsc')

        # change dir to alter local path
        cwd = os.getcwd()
        os.chdir(self.root)

        tmp_wd = os.getcwd()

        expected = get_joined_path(tmp_wd, 'avro', 'nest', 'test2.avsc')

        path1 = get_system_path(get_joined_path(self.nest_dir, 'test2.avsc'))

        # change dir to alter local path
        cwd = os.getcwd()
        os.chdir(self.nest_dir)
        print(os.getcwd())

        path2 = get_system_path('test2.avsc')

        # change back to original dir to prevent errors
        os.chdir(cwd)

        self.assertEqual(path1, path2, f'paths not equal: {path1} != {path2}')

        self.assertEqual(expected, path2,
                         f'paths not equal: {expected} != {path2}')

    def test_verify_or_create_namespace_path(self):
        """ tests that verify_or_create_namespace_path works """
        rootdir = self.root
        namespace = 'test.namespace.path'

        verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)

        self.assertTrue(
            verify_path_exists(
                get_joined_path(self.test_dir, 'namespace', 'path')),
            'should have created the path from a namespace')

        verify_or_create_namespace_path(rootdir=rootdir, namespace=namespace)

        self.assertTrue(
            verify_path_exists(
                get_joined_path(self.test_dir, 'namespace', 'path')),
            'should have created the path from a namespace')
        shutil.rmtree(self.test_dir)

    def test_get_or_create_path(self):
        """ tests that get_or_create_path works """
        path = self.test_dir

        get_or_create_path(path=path)

        self.assertTrue(verify_path_exists(self.test_dir),
                        'should have created path at /tmp/test')
        os.rmdir(path)

        os.chdir(self.root)

        get_or_create_path(path='test')

        self.assertTrue(
            verify_path_exists(self.test_dir),
            'should have created path at /tmp/test from relative path')
        shutil.rmtree(path)