def test_can_change_data_path_after_creation(self):
     original_path = test_helpers.fixture_path('unicode.txt')
     new_path = test_helpers.fixture_path('foo.txt')
     resource_dict = {'path': original_path}
     resource = datapackage.Resource.load(resource_dict)
     resource.descriptor['path'] = new_path
     assert resource.data == b'foo\n'
Example #2
0
 def test_can_change_data_path_after_creation(self):
     original_path = test_helpers.fixture_path('unicode.txt')
     new_path = test_helpers.fixture_path('foo.txt')
     resource_dict = {
         'path': original_path
     }
     resource = datapackage.Resource.load(resource_dict)
     resource.descriptor['path'] = new_path
     assert resource.data == b'foo\n'
 def test_generates_unique_filenames_for_unnamed_resources(self, tmpfile):
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'path': test_helpers.fixture_path('unicode.txt')},
             {'path': test_helpers.fixture_path('foo.txt')}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         files = z.namelist()
         assert sorted(set(files)) == sorted(files)
 def test_generates_unique_filenames_for_unnamed_resources(self, tmpfile):
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'path': test_helpers.fixture_path('unicode.txt')},
             {'path': test_helpers.fixture_path('foo.txt')}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         files = z.namelist()
         assert sorted(set(files)) == sorted(files)
 def test_init_raises_if_path_is_a_bad_json(self):
     bad_json = test_helpers.fixture_path('bad_json.json')
     with pytest.raises(datapackage.exceptions.DataPackageException) as excinfo:
         datapackage.DataPackage(bad_json)
     message = str(excinfo.value)
     assert 'Unable to parse JSON' in message
     assert 'line 2 column 5 (char 6)' in message
Example #6
0
 def test_works_with_resources_with_relative_paths(self, tmpfile):
     path = test_helpers.fixture_path(
         'datapackage_with_foo.txt_resource.json')
     dp = datapackage.DataPackage(path)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         assert len(z.filelist) == 2
Example #7
0
    def test_get_raises_if_profile_isnt_a_json(self):
        registry_path = test_helpers.fixture_path(
            'registry_with_notajson_profile.json')
        registry = datapackage.registry.Registry(registry_path)

        with pytest.raises(RegistryError):
            registry.get('notajson-data-package')
Example #8
0
 def test_remote_data_path_returns_none_if_theres_no_remote_data(self):
     resource_dict = {
         'data': 'foo',
         'path': test_helpers.fixture_path('unicode.txt'),
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.remote_data_path is None
Example #9
0
 def test_init_raises_if_path_is_a_bad_json(self):
     bad_json = test_helpers.fixture_path('bad_json.json')
     with pytest.raises(datapackage.exceptions.DataPackageException) as excinfo:
         datapackage.DataPackage(bad_json)
     message = str(excinfo.value)
     assert 'Unable to parse JSON' in message
     assert 'line 2 column 5 (char 6)' in message
Example #10
0
 def test_remote_data_path_returns_none_if_theres_no_remote_data(self):
     resource_dict = {
         'data': 'foo',
         'path': test_helpers.fixture_path('unicode.txt'),
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.remote_data_path is None
Example #11
0
def test_push_datapackage(storage):

    # Prepare and call
    descriptor = helpers.fixture_path('datapackage', 'datapackage.json')
    storage.buckets = ['data___data']  # Without patch it's a reflection
    module.push_datapackage(descriptor=descriptor, backend='backend')

    # Assert mocked calls
    storage.create.assert_called_with(['data___data'], [{
        'fields': [{
            'name': 'id',
            'type': 'integer'
        }, {
            'name': 'city',
            'type': 'string'
        }]
    }])
    storage.write.assert_called_with('data___data', ANY)

    # Assert writen data
    data = storage.write.call_args[0][1]
    assert list(data) == [
        (1, 'London'),
        (2, 'Paris'),
    ]
Example #12
0
 def test_load_accepts_absolute_paths(self):
     path = test_helpers.fixture_path('foo.txt')
     resource_dict = {
         'path': path,
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.data == b'foo\n'
Example #13
0
 def test_load_accepts_absolute_paths(self):
     path = test_helpers.fixture_path('foo.txt')
     resource_dict = {
         'path': path,
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.data == b'foo\n'
Example #14
0
    def test_load_binary_data(self):
        resource_dict = {
            'path': test_helpers.fixture_path('image.gif'),
        }
        resource = datapackage.Resource.load(resource_dict)

        with open(resource_dict['path'], 'rb') as f:
            assert resource.data == f.read()
Example #15
0
    def test_local_data_path(self, datapackage_zip):
        dp = datapackage.DataPackage(datapackage_zip)

        assert dp.resources[0].local_data_path is not None

        with open(test_helpers.fixture_path('foo.txt')) as data_file:
            with open(dp.resources[0].local_data_path) as local_data_file:
                assert local_data_file.read() == data_file.read()
Example #16
0
 def test_with_local_resources_with_existent_path_isnt_safe(self):
     descriptor = {
         'resources': [
             {'path': test_helpers.fixture_path('foo.txt')},
         ]
     }
     with pytest.raises(datapackage.exceptions.DataPackageException):
         dp = datapackage.DataPackage(descriptor, {})
 def test_with_local_resources_with_existent_path_isnt_safe(self):
     metadata = {
         'resources': [
             {'path': test_helpers.fixture_path('foo.txt')},
         ]
     }
     dp = datapackage.DataPackage(metadata, {})
     assert not dp.safe()
    def test_local_data_path(self, datapackage_zip):
        dp = datapackage.DataPackage(datapackage_zip)

        assert dp.resources[0].local_data_path is not None

        with open(test_helpers.fixture_path('foo.txt')) as data_file:
            with open(dp.resources[0].local_data_path) as local_data_file:
                assert local_data_file.read() == data_file.read()
 def test_works_with_resources_with_relative_paths(self, tmpfile):
     path = test_helpers.fixture_path(
         'datapackage_with_foo.txt_resource.json'
     )
     dp = datapackage.DataPackage(path)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         assert len(z.filelist) == 2
Example #20
0
 def test_load_prefers_loading_local_data_over_url(self):
     httpretty.HTTPretty.allow_net_connect = False
     resource_dict = {
         'path': test_helpers.fixture_path('foo.txt'),
         'url': 'http://someplace.com/inexistent-file.txt',
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.data == b'foo\n'
Example #21
0
 def test_load_prefers_loading_local_data_over_url(self):
     httpretty.HTTPretty.allow_net_connect = False
     resource_dict = {
         'path': test_helpers.fixture_path('foo.txt'),
         'url': 'http://someplace.com/inexistent-file.txt',
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.data == b'foo\n'
Example #22
0
    def _create_resource_file_with(self, fixture):
        path = test_helpers.fixture_path(fixture)
        with open(path, 'rb') as f:
            body = f.read()
        url = 'http://www.someplace.com/{fixture}'.format(fixture=fixture)
        httpretty.register_uri(httpretty.GET, url, body=body)

        return RemoteResourceFile(url)
Example #23
0
    def test_load_binary_data(self):
        resource_dict = {
            'path': test_helpers.fixture_path('image.gif'),
        }
        resource = datapackage.Resource.load(resource_dict)

        with open(resource_dict['path'], 'rb') as f:
            assert resource.data == f.read()
 def test_load_accepts_relative_paths(self):
     filename = 'foo.txt'
     base_path = os.path.dirname(test_helpers.fixture_path(filename))
     resource_dict = {
         'path': filename,
     }
     resource = datapackage.Resource.load(resource_dict, base_path)
     assert resource.data == b'foo\n'
Example #25
0
 def test_local_data_path_returns_the_absolute_path(self):
     base_path = test_helpers.fixture_path('')
     path = os.path.join(base_path, '..', 'fixtures', 'unicode.txt')
     resource_dict = {
         'path': path,
     }
     resource = datapackage.Resource.load(resource_dict)
     abs_path = os.path.join(base_path, 'unicode.txt')
     assert resource.local_data_path == abs_path
Example #26
0
 def test_local_data_path_returns_the_absolute_path_relative_to_base(self):
     base_path = test_helpers.fixture_path('')
     resource_dict = {
         'path': 'unicode.txt',
         'base': base_path,
     }
     resource = datapackage.Resource.load(resource_dict)
     abs_path = os.path.join(base_path, resource_dict['path'])
     assert resource.local_data_path == abs_path
Example #27
0
 def test_local_data_path_returns_the_absolute_path(self):
     base_path = test_helpers.fixture_path('')
     path = os.path.join(base_path, '..', 'fixtures', 'unicode.txt')
     resource_dict = {
         'path': path,
     }
     resource = datapackage.Resource.load(resource_dict)
     abs_path = os.path.join(base_path, 'unicode.txt')
     assert resource.local_data_path == abs_path
    def _create_resource_file_with(self, fixture):
        path = test_helpers.fixture_path(fixture)
        with open(path, 'rb') as f:
            body = f.read()
        url = 'http://www.someplace.com/{fixture}'.format(fixture=fixture)
        httpretty.register_uri(httpretty.GET,
                               url,
                               body=body)

        return RemoteResourceFile(url)
Example #29
0
 def test_load_accepts_relative_paths(self):
     filename = 'foo.txt'
     base_path = os.path.dirname(
         test_helpers.fixture_path(filename)
     )
     resource_dict = {
         'path': filename,
     }
     resource = datapackage.Resource.load(resource_dict, base_path)
     assert resource.data == b'foo\n'
def datapackage_zip(tmpfile):
    metadata = {
        'name': 'proverbs',
        'resources': [
            {'path': test_helpers.fixture_path('foo.txt')},
        ]
    }
    dp = datapackage.DataPackage(metadata)
    dp.save(tmpfile)
    return tmpfile
Example #31
0
 def test_local_resource_with_absolute_path_is_loaded(self):
     path = test_helpers.fixture_path('foo.txt')
     metadata = {
         'resources': [
             {'path': path},
         ],
     }
     dp = datapackage.DataPackage(metadata)
     assert len(dp.resources) == 1
     assert dp.resources[0].data == b'foo\n'
Example #32
0
 def test_with_local_resources_with_existent_path_isnt_safe(self):
     descriptor = {
         'resources': [
             {
                 'path': test_helpers.fixture_path('foo.txt')
             },
         ]
     }
     dp = datapackage.DataPackage(descriptor, {})
     assert not dp.safe()
 def test_local_resource_with_absolute_path_is_loaded(self):
     path = test_helpers.fixture_path('foo.txt')
     metadata = {
         'resources': [
             {'path': path},
         ],
     }
     dp = datapackage.DataPackage(metadata)
     assert len(dp.resources) == 1
     assert dp.resources[0].data == b'foo\n'
Example #34
0
def datapackage_zip():
    with tempfile.NamedTemporaryFile() as f:
        metadata = {
            'name': 'proverbs',
            'resources': [
                {'path': test_helpers.fixture_path('foo.txt')},
            ]
        }
        dp = datapackage.DataPackage(metadata)
        dp.save(f)
        yield f
Example #35
0
 def test_load_accepts_relative_paths_with_base_defined_in_metadata(self):
     filename = 'foo.txt'
     base_path = os.path.dirname(
         test_helpers.fixture_path(filename)
     )
     resource_dict = {
         'path': filename,
         'base': base_path,
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.data == b'foo\n'
def datapackage_zip(tmpfile):
    metadata = {
        'name': 'proverbs',
        'resources': [
            {
                'path': test_helpers.fixture_path('foo.txt')
            },
        ]
    }
    dp = datapackage.DataPackage(metadata)
    dp.save(tmpfile)
    return tmpfile
Example #37
0
 def test_load_base_path_in_metadata_overloads_base_passed_in_args(self):
     filename = 'foo.txt'
     base_path = os.path.dirname(
         test_helpers.fixture_path(filename)
     )
     resource_dict = {
         'path': filename,
         'base': base_path,
     }
     resource = datapackage.Resource.load(resource_dict,
                                          'invalid_base_path')
     assert resource.data == b'foo\n'
Example #38
0
 def test_works_with_resources_with_relative_paths(self, tmpfile):
     base_path = test_helpers.fixture_path('')
     metadata = {
         'name': 'proverbs',
         'base': base_path,
         'resources': [
             {'path': 'unicode.txt'}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
 def test_adds_resources_inside_data_subfolder(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {'name': 'proverbs', 'resources': [{'path': resource_path}]}
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         filename = [
             name for name in z.namelist() if name.startswith('data/')
         ]
         assert len(filename) == 1
         resource_data = z.read(filename[0]).decode('utf-8')
     assert resource_data == '万事开头难\n'
 def test_generates_filenames_for_named_resources(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'name': 'proverbs', 'format': 'TXT', 'path': resource_path},
             {'name': 'proverbs_without_format', 'path': resource_path}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         assert 'data/proverbs.txt' in z.namelist()
         assert 'data/proverbs_without_format' in z.namelist()
 def test_fixes_resources_paths_to_be_relative_to_package(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'name': 'unicode', 'format': 'txt', 'path': resource_path}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         json_string = z.read('datapackage.json').decode('utf-8')
         generated_dp_dict = json.loads(json_string)
     assert generated_dp_dict['resources'][0]['path'] == 'data/unicode.txt'
Example #42
0
 def test_fixes_resources_paths_to_be_relative_to_package(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'name': 'unicode', 'format': 'txt', 'path': resource_path}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         json_string = z.read('datapackage.json').decode('utf-8')
         generated_dp_dict = json.loads(json_string)
     assert generated_dp_dict['resources'][0]['path'] == 'data/unicode.txt'
Example #43
0
 def test_generates_filenames_for_named_resources(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'name': 'proverbs', 'format': 'TXT', 'path': resource_path},
             {'name': 'proverbs_without_format', 'path': resource_path}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         assert 'data/proverbs.txt' in z.namelist()
         assert 'data/proverbs_without_format' in z.namelist()
 def test_adds_resources_inside_data_subfolder(self, tmpfile):
     resource_path = test_helpers.fixture_path('unicode.txt')
     metadata = {
         'name': 'proverbs',
         'resources': [
             {'path': resource_path}
         ]
     }
     schema = {}
     dp = datapackage.DataPackage(metadata, schema)
     dp.save(tmpfile)
     with zipfile.ZipFile(tmpfile, 'r') as z:
         filename = [name for name in z.namelist()
                     if name.startswith('data/')]
         assert len(filename) == 1
         resource_data = z.read(filename[0]).decode('utf-8')
     assert resource_data == '万事开头难\n'
Example #45
0
    def test_load_tsv(self):
        resource_dict = {
            'path': test_helpers.fixture_path('cities.tsv')
        }

        resource = TabularResource(resource_dict)
        assert resource.data == [
            {'Area': '1807.92', 'Name': 'Acrelândia', 'Population': '12538', 'State': 'AC'},
            {'Area': '186.53', 'Name': 'Boca da Mata', 'Population': '25776', 'State': 'AL'},
            {'Area': '242.62', 'Name': 'Capela', 'Population': '17077', 'State': 'AL'},
            {'Area': '6709.66', 'Name': 'Tartarugalzinho', 'Population': '12563', 'State': 'AP'},
            {'Area': '837.72', 'Name': 'América Dourada', 'Population': None, 'State': 'BA'},
            {'Area': '204.79', 'Name': 'Jijoca de Jericoacoara', 'Population': '17002', 'State': 'CE'},
            {'Area': '6953.67', 'Name': 'Cavalcante', 'Population': '9392', 'State': 'GO'},
            {'Area': '8258.42', 'Name': 'Centro Novo do Maranhão', 'Population': '17622', 'State': 'MA'},
            {'Area': '3651.18', 'Name': 'Ped\\ro G\\omes', 'Population': '7967', 'State': 'MS'},
            {'Area': '881.06', 'Name': 'Abadia dos Dourados', 'Population': '6704', 'State': 'MG'},
        ]
Example #46
0
    def test_load_tsv(self):
        resource_dict = {
            'path': test_helpers.fixture_path('cities.tsv')
        }

        resource = TabularResource(resource_dict)
        assert resource.data == [
            {'Area': '1807.92', 'Name': 'Acrelândia', 'Population': '12538', 'State': 'AC'},
            {'Area': '186.53', 'Name': 'Boca da Mata', 'Population': '25776', 'State': 'AL'},
            {'Area': '242.62', 'Name': 'Capela', 'Population': '17077', 'State': 'AL'},
            {'Area': '6709.66', 'Name': 'Tartarugalzinho', 'Population': '12563', 'State': 'AP'},
            {'Area': '837.72', 'Name': 'América Dourada', 'Population': None, 'State': 'BA'},
            {'Area': '204.79', 'Name': 'Jijoca de Jericoacoara', 'Population': '17002', 'State': 'CE'},
            {'Area': '6953.67', 'Name': 'Cavalcante', 'Population': '9392', 'State': 'GO'},
            {'Area': '8258.42', 'Name': 'Centro Novo do Maranhão', 'Population': '17622', 'State': 'MA'},
            {'Area': '3651.18', 'Name': 'Ped\\ro G\\omes', 'Population': '7967', 'State': 'MS'},
            {'Area': '881.06', 'Name': 'Abadia dos Dourados', 'Population': '6704', 'State': 'MG'},
        ]
Example #47
0
def test_push_datapackage(storage):

    # Prepare and call
    descriptor = helpers.fixture_path('datapackage', 'datapackage.json')
    storage.tables = ['data___data']  # Without patch it's a reflection
    module.push_datapackage(descriptor=descriptor, backend='backend')

    # Assert mocked calls
    storage.create.assert_called_with(
        ['data___data'],
        [{'fields': [
            {'name': 'id', 'type': 'integer'},
            {'name': 'city', 'type': 'string'}]}])
    storage.write.assert_called_with('data___data', ANY)

    # Assert writen data
    data = storage.write.call_args[0][1]
    assert list(data) == [
        (1, 'London'),
        (2, 'Paris'),
    ]
 def test_local_with_relative_resources_paths_is_safe(self):
     fixture_name = 'datapackage_with_foo.txt_resource.json'
     path = test_helpers.fixture_path(fixture_name)
     dp = datapackage.DataPackage(path, {})
     assert dp.safe()
 def test_base_path_is_datapackages_base_path_when_it_is_a_file(self):
     path = test_helpers.fixture_path('empty_datapackage.json')
     base_path = os.path.dirname(path)
     dp = datapackage.DataPackage(path)
     assert dp.base_path == base_path
 def test_init_raises_if_path_json_isnt_a_dict(self):
     empty_array_path = test_helpers.fixture_path('empty_array.json')
     with pytest.raises(datapackage.exceptions.DataPackageException):
         datapackage.DataPackage(empty_array_path)
 def test_init_raises_if_path_isnt_a_json(self):
     not_a_json_path = test_helpers.fixture_path('not_a_json')
     with pytest.raises(datapackage_validate.exceptions.SchemaError):
         Schema(not_a_json_path)
 def test_init_raises_if_path_isnt_a_json(self):
     not_a_json_path = test_helpers.fixture_path('not_a_json')
     with pytest.raises(datapackage.exceptions.DataPackageException):
         datapackage.DataPackage(not_a_json_path)
Example #53
0
 def test_init_raises_if_path_isnt_a_json(self):
     not_a_json_path = test_helpers.fixture_path('not_a_json')
     with pytest.raises(datapackage.exceptions.DataPackageException):
         datapackage.DataPackage(not_a_json_path)
Example #54
0
 def test_init_raises_if_path_json_isnt_a_dict(self):
     empty_array_path = test_helpers.fixture_path('empty_array.json')
     with pytest.raises(datapackage.exceptions.DataPackageException):
         datapackage.DataPackage(empty_array_path)
 def test_init_loads_schema_from_path(self):
     schema_path = test_helpers.fixture_path('empty_schema.json')
     assert Schema(schema_path).to_dict() == {}
Example #56
0
 def test_local_with_relative_resources_paths_is_safe(self):
     fixture_name = 'datapackage_with_foo.txt_resource.json'
     path = test_helpers.fixture_path(fixture_name)
     dp = datapackage.DataPackage(path, {})
     assert dp.safe()
 def test_init_accepts_file_paths(self):
     path = test_helpers.fixture_path('empty_datapackage.json')
     dp = datapackage.DataPackage(path)
     assert dp.metadata == {}
    def test_raises_if_profile_in_local_file_isnt_a_json(self):
        registry_path = test_helpers.fixture_path('registry_with_notajson_profile.csv')
        registry = datapackage_registry.Registry(registry_path)

        with assert_raises(DataPackageRegistryException):
            registry.get('notajson')
Example #59
0
 def test_local_data_path_returns_the_unmodified_path(self):
     resource_dict = {
         'path': test_helpers.fixture_path('unicode.txt'),
     }
     resource = datapackage.Resource.load(resource_dict)
     assert resource.local_data_path == resource_dict['path']
 def test_local_resource_with_relative_path_is_loaded(self):
     datapackage_filename = 'datapackage_with_foo.txt_resource.json'
     path = test_helpers.fixture_path(datapackage_filename)
     dp = datapackage.DataPackage(path)
     assert len(dp.resources) == 1
     assert dp.resources[0].data == b'foo\n'