def test_can_change_data_path_after_creation(self): original_path = test_helpers.fixture_path('unicode.txt') new_path = test_helpers.fixture_path('foo.txt') resource_dict = {'path': original_path} resource = datapackage.Resource.load(resource_dict) resource.descriptor['path'] = new_path assert resource.data == b'foo\n'
def test_can_change_data_path_after_creation(self): original_path = test_helpers.fixture_path('unicode.txt') new_path = test_helpers.fixture_path('foo.txt') resource_dict = { 'path': original_path } resource = datapackage.Resource.load(resource_dict) resource.descriptor['path'] = new_path assert resource.data == b'foo\n'
def test_generates_unique_filenames_for_unnamed_resources(self, tmpfile): metadata = { 'name': 'proverbs', 'resources': [ {'path': test_helpers.fixture_path('unicode.txt')}, {'path': test_helpers.fixture_path('foo.txt')} ] } schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: files = z.namelist() assert sorted(set(files)) == sorted(files)
def test_init_raises_if_path_is_a_bad_json(self): bad_json = test_helpers.fixture_path('bad_json.json') with pytest.raises(datapackage.exceptions.DataPackageException) as excinfo: datapackage.DataPackage(bad_json) message = str(excinfo.value) assert 'Unable to parse JSON' in message assert 'line 2 column 5 (char 6)' in message
def test_works_with_resources_with_relative_paths(self, tmpfile): path = test_helpers.fixture_path( 'datapackage_with_foo.txt_resource.json') dp = datapackage.DataPackage(path) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: assert len(z.filelist) == 2
def test_get_raises_if_profile_isnt_a_json(self): registry_path = test_helpers.fixture_path( 'registry_with_notajson_profile.json') registry = datapackage.registry.Registry(registry_path) with pytest.raises(RegistryError): registry.get('notajson-data-package')
def test_remote_data_path_returns_none_if_theres_no_remote_data(self): resource_dict = { 'data': 'foo', 'path': test_helpers.fixture_path('unicode.txt'), } resource = datapackage.Resource.load(resource_dict) assert resource.remote_data_path is None
def test_push_datapackage(storage): # Prepare and call descriptor = helpers.fixture_path('datapackage', 'datapackage.json') storage.buckets = ['data___data'] # Without patch it's a reflection module.push_datapackage(descriptor=descriptor, backend='backend') # Assert mocked calls storage.create.assert_called_with(['data___data'], [{ 'fields': [{ 'name': 'id', 'type': 'integer' }, { 'name': 'city', 'type': 'string' }] }]) storage.write.assert_called_with('data___data', ANY) # Assert writen data data = storage.write.call_args[0][1] assert list(data) == [ (1, 'London'), (2, 'Paris'), ]
def test_load_accepts_absolute_paths(self): path = test_helpers.fixture_path('foo.txt') resource_dict = { 'path': path, } resource = datapackage.Resource.load(resource_dict) assert resource.data == b'foo\n'
def test_load_binary_data(self): resource_dict = { 'path': test_helpers.fixture_path('image.gif'), } resource = datapackage.Resource.load(resource_dict) with open(resource_dict['path'], 'rb') as f: assert resource.data == f.read()
def test_local_data_path(self, datapackage_zip): dp = datapackage.DataPackage(datapackage_zip) assert dp.resources[0].local_data_path is not None with open(test_helpers.fixture_path('foo.txt')) as data_file: with open(dp.resources[0].local_data_path) as local_data_file: assert local_data_file.read() == data_file.read()
def test_with_local_resources_with_existent_path_isnt_safe(self): descriptor = { 'resources': [ {'path': test_helpers.fixture_path('foo.txt')}, ] } with pytest.raises(datapackage.exceptions.DataPackageException): dp = datapackage.DataPackage(descriptor, {})
def test_with_local_resources_with_existent_path_isnt_safe(self): metadata = { 'resources': [ {'path': test_helpers.fixture_path('foo.txt')}, ] } dp = datapackage.DataPackage(metadata, {}) assert not dp.safe()
def test_works_with_resources_with_relative_paths(self, tmpfile): path = test_helpers.fixture_path( 'datapackage_with_foo.txt_resource.json' ) dp = datapackage.DataPackage(path) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: assert len(z.filelist) == 2
def test_load_prefers_loading_local_data_over_url(self): httpretty.HTTPretty.allow_net_connect = False resource_dict = { 'path': test_helpers.fixture_path('foo.txt'), 'url': 'http://someplace.com/inexistent-file.txt', } resource = datapackage.Resource.load(resource_dict) assert resource.data == b'foo\n'
def _create_resource_file_with(self, fixture): path = test_helpers.fixture_path(fixture) with open(path, 'rb') as f: body = f.read() url = 'http://www.someplace.com/{fixture}'.format(fixture=fixture) httpretty.register_uri(httpretty.GET, url, body=body) return RemoteResourceFile(url)
def test_load_accepts_relative_paths(self): filename = 'foo.txt' base_path = os.path.dirname(test_helpers.fixture_path(filename)) resource_dict = { 'path': filename, } resource = datapackage.Resource.load(resource_dict, base_path) assert resource.data == b'foo\n'
def test_local_data_path_returns_the_absolute_path(self): base_path = test_helpers.fixture_path('') path = os.path.join(base_path, '..', 'fixtures', 'unicode.txt') resource_dict = { 'path': path, } resource = datapackage.Resource.load(resource_dict) abs_path = os.path.join(base_path, 'unicode.txt') assert resource.local_data_path == abs_path
def test_local_data_path_returns_the_absolute_path_relative_to_base(self): base_path = test_helpers.fixture_path('') resource_dict = { 'path': 'unicode.txt', 'base': base_path, } resource = datapackage.Resource.load(resource_dict) abs_path = os.path.join(base_path, resource_dict['path']) assert resource.local_data_path == abs_path
def test_load_accepts_relative_paths(self): filename = 'foo.txt' base_path = os.path.dirname( test_helpers.fixture_path(filename) ) resource_dict = { 'path': filename, } resource = datapackage.Resource.load(resource_dict, base_path) assert resource.data == b'foo\n'
def datapackage_zip(tmpfile): metadata = { 'name': 'proverbs', 'resources': [ {'path': test_helpers.fixture_path('foo.txt')}, ] } dp = datapackage.DataPackage(metadata) dp.save(tmpfile) return tmpfile
def test_local_resource_with_absolute_path_is_loaded(self): path = test_helpers.fixture_path('foo.txt') metadata = { 'resources': [ {'path': path}, ], } dp = datapackage.DataPackage(metadata) assert len(dp.resources) == 1 assert dp.resources[0].data == b'foo\n'
def test_with_local_resources_with_existent_path_isnt_safe(self): descriptor = { 'resources': [ { 'path': test_helpers.fixture_path('foo.txt') }, ] } dp = datapackage.DataPackage(descriptor, {}) assert not dp.safe()
def datapackage_zip(): with tempfile.NamedTemporaryFile() as f: metadata = { 'name': 'proverbs', 'resources': [ {'path': test_helpers.fixture_path('foo.txt')}, ] } dp = datapackage.DataPackage(metadata) dp.save(f) yield f
def test_load_accepts_relative_paths_with_base_defined_in_metadata(self): filename = 'foo.txt' base_path = os.path.dirname( test_helpers.fixture_path(filename) ) resource_dict = { 'path': filename, 'base': base_path, } resource = datapackage.Resource.load(resource_dict) assert resource.data == b'foo\n'
def datapackage_zip(tmpfile): metadata = { 'name': 'proverbs', 'resources': [ { 'path': test_helpers.fixture_path('foo.txt') }, ] } dp = datapackage.DataPackage(metadata) dp.save(tmpfile) return tmpfile
def test_load_base_path_in_metadata_overloads_base_passed_in_args(self): filename = 'foo.txt' base_path = os.path.dirname( test_helpers.fixture_path(filename) ) resource_dict = { 'path': filename, 'base': base_path, } resource = datapackage.Resource.load(resource_dict, 'invalid_base_path') assert resource.data == b'foo\n'
def test_works_with_resources_with_relative_paths(self, tmpfile): base_path = test_helpers.fixture_path('') metadata = { 'name': 'proverbs', 'base': base_path, 'resources': [ {'path': 'unicode.txt'} ] } schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile)
def test_adds_resources_inside_data_subfolder(self, tmpfile): resource_path = test_helpers.fixture_path('unicode.txt') metadata = {'name': 'proverbs', 'resources': [{'path': resource_path}]} schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: filename = [ name for name in z.namelist() if name.startswith('data/') ] assert len(filename) == 1 resource_data = z.read(filename[0]).decode('utf-8') assert resource_data == '万事开头难\n'
def test_generates_filenames_for_named_resources(self, tmpfile): resource_path = test_helpers.fixture_path('unicode.txt') metadata = { 'name': 'proverbs', 'resources': [ {'name': 'proverbs', 'format': 'TXT', 'path': resource_path}, {'name': 'proverbs_without_format', 'path': resource_path} ] } schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: assert 'data/proverbs.txt' in z.namelist() assert 'data/proverbs_without_format' in z.namelist()
def test_fixes_resources_paths_to_be_relative_to_package(self, tmpfile): resource_path = test_helpers.fixture_path('unicode.txt') metadata = { 'name': 'proverbs', 'resources': [ {'name': 'unicode', 'format': 'txt', 'path': resource_path} ] } schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: json_string = z.read('datapackage.json').decode('utf-8') generated_dp_dict = json.loads(json_string) assert generated_dp_dict['resources'][0]['path'] == 'data/unicode.txt'
def test_adds_resources_inside_data_subfolder(self, tmpfile): resource_path = test_helpers.fixture_path('unicode.txt') metadata = { 'name': 'proverbs', 'resources': [ {'path': resource_path} ] } schema = {} dp = datapackage.DataPackage(metadata, schema) dp.save(tmpfile) with zipfile.ZipFile(tmpfile, 'r') as z: filename = [name for name in z.namelist() if name.startswith('data/')] assert len(filename) == 1 resource_data = z.read(filename[0]).decode('utf-8') assert resource_data == '万事开头难\n'
def test_load_tsv(self): resource_dict = { 'path': test_helpers.fixture_path('cities.tsv') } resource = TabularResource(resource_dict) assert resource.data == [ {'Area': '1807.92', 'Name': 'Acrelândia', 'Population': '12538', 'State': 'AC'}, {'Area': '186.53', 'Name': 'Boca da Mata', 'Population': '25776', 'State': 'AL'}, {'Area': '242.62', 'Name': 'Capela', 'Population': '17077', 'State': 'AL'}, {'Area': '6709.66', 'Name': 'Tartarugalzinho', 'Population': '12563', 'State': 'AP'}, {'Area': '837.72', 'Name': 'América Dourada', 'Population': None, 'State': 'BA'}, {'Area': '204.79', 'Name': 'Jijoca de Jericoacoara', 'Population': '17002', 'State': 'CE'}, {'Area': '6953.67', 'Name': 'Cavalcante', 'Population': '9392', 'State': 'GO'}, {'Area': '8258.42', 'Name': 'Centro Novo do Maranhão', 'Population': '17622', 'State': 'MA'}, {'Area': '3651.18', 'Name': 'Ped\\ro G\\omes', 'Population': '7967', 'State': 'MS'}, {'Area': '881.06', 'Name': 'Abadia dos Dourados', 'Population': '6704', 'State': 'MG'}, ]
def test_push_datapackage(storage): # Prepare and call descriptor = helpers.fixture_path('datapackage', 'datapackage.json') storage.tables = ['data___data'] # Without patch it's a reflection module.push_datapackage(descriptor=descriptor, backend='backend') # Assert mocked calls storage.create.assert_called_with( ['data___data'], [{'fields': [ {'name': 'id', 'type': 'integer'}, {'name': 'city', 'type': 'string'}]}]) storage.write.assert_called_with('data___data', ANY) # Assert writen data data = storage.write.call_args[0][1] assert list(data) == [ (1, 'London'), (2, 'Paris'), ]
def test_local_with_relative_resources_paths_is_safe(self): fixture_name = 'datapackage_with_foo.txt_resource.json' path = test_helpers.fixture_path(fixture_name) dp = datapackage.DataPackage(path, {}) assert dp.safe()
def test_base_path_is_datapackages_base_path_when_it_is_a_file(self): path = test_helpers.fixture_path('empty_datapackage.json') base_path = os.path.dirname(path) dp = datapackage.DataPackage(path) assert dp.base_path == base_path
def test_init_raises_if_path_json_isnt_a_dict(self): empty_array_path = test_helpers.fixture_path('empty_array.json') with pytest.raises(datapackage.exceptions.DataPackageException): datapackage.DataPackage(empty_array_path)
def test_init_raises_if_path_isnt_a_json(self): not_a_json_path = test_helpers.fixture_path('not_a_json') with pytest.raises(datapackage_validate.exceptions.SchemaError): Schema(not_a_json_path)
def test_init_raises_if_path_isnt_a_json(self): not_a_json_path = test_helpers.fixture_path('not_a_json') with pytest.raises(datapackage.exceptions.DataPackageException): datapackage.DataPackage(not_a_json_path)
def test_init_loads_schema_from_path(self): schema_path = test_helpers.fixture_path('empty_schema.json') assert Schema(schema_path).to_dict() == {}
def test_init_accepts_file_paths(self): path = test_helpers.fixture_path('empty_datapackage.json') dp = datapackage.DataPackage(path) assert dp.metadata == {}
def test_raises_if_profile_in_local_file_isnt_a_json(self): registry_path = test_helpers.fixture_path('registry_with_notajson_profile.csv') registry = datapackage_registry.Registry(registry_path) with assert_raises(DataPackageRegistryException): registry.get('notajson')
def test_local_data_path_returns_the_unmodified_path(self): resource_dict = { 'path': test_helpers.fixture_path('unicode.txt'), } resource = datapackage.Resource.load(resource_dict) assert resource.local_data_path == resource_dict['path']
def test_local_resource_with_relative_path_is_loaded(self): datapackage_filename = 'datapackage_with_foo.txt_resource.json' path = test_helpers.fixture_path(datapackage_filename) dp = datapackage.DataPackage(path) assert len(dp.resources) == 1 assert dp.resources[0].data == b'foo\n'