def test_can_change_data_path_after_creation(self):
original_path = test_helpers.fixture_path('unicode.txt')
new_path = test_helpers.fixture_path('foo.txt')
resource_dict = {'path': original_path}
resource = datapackage.Resource.load(resource_dict)
resource.descriptor['path'] = new_path
assert resource.data == b'foo\n'
def test_can_change_data_path_after_creation(self):
original_path = test_helpers.fixture_path('unicode.txt')
new_path = test_helpers.fixture_path('foo.txt')
resource_dict = {
'path': original_path
}
resource = datapackage.Resource.load(resource_dict)
resource.descriptor['path'] = new_path
assert resource.data == b'foo\n'
def test_generates_unique_filenames_for_unnamed_resources(self, tmpfile):
metadata = {
'name': 'proverbs',
'resources': [
{'path': test_helpers.fixture_path('unicode.txt')},
{'path': test_helpers.fixture_path('foo.txt')}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
files = z.namelist()
assert sorted(set(files)) == sorted(files)
def test_generates_unique_filenames_for_unnamed_resources(self, tmpfile):
metadata = {
'name': 'proverbs',
'resources': [
{'path': test_helpers.fixture_path('unicode.txt')},
{'path': test_helpers.fixture_path('foo.txt')}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
files = z.namelist()
assert sorted(set(files)) == sorted(files)
def test_init_raises_if_path_is_a_bad_json(self):
bad_json = test_helpers.fixture_path('bad_json.json')
with pytest.raises(datapackage.exceptions.DataPackageException) as excinfo:
datapackage.DataPackage(bad_json)
message = str(excinfo.value)
assert 'Unable to parse JSON' in message
assert 'line 2 column 5 (char 6)' in message
def test_works_with_resources_with_relative_paths(self, tmpfile):
path = test_helpers.fixture_path(
'datapackage_with_foo.txt_resource.json')
dp = datapackage.DataPackage(path)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
assert len(z.filelist) == 2
def test_get_raises_if_profile_isnt_a_json(self):
registry_path = test_helpers.fixture_path(
'registry_with_notajson_profile.json')
registry = datapackage.registry.Registry(registry_path)
with pytest.raises(RegistryError):
registry.get('notajson-data-package')
def test_remote_data_path_returns_none_if_theres_no_remote_data(self):
resource_dict = {
'data': 'foo',
'path': test_helpers.fixture_path('unicode.txt'),
}
resource = datapackage.Resource.load(resource_dict)
assert resource.remote_data_path is None
def test_init_raises_if_path_is_a_bad_json(self):
bad_json = test_helpers.fixture_path('bad_json.json')
with pytest.raises(datapackage.exceptions.DataPackageException) as excinfo:
datapackage.DataPackage(bad_json)
message = str(excinfo.value)
assert 'Unable to parse JSON' in message
assert 'line 2 column 5 (char 6)' in message
def test_remote_data_path_returns_none_if_theres_no_remote_data(self):
resource_dict = {
'data': 'foo',
'path': test_helpers.fixture_path('unicode.txt'),
}
resource = datapackage.Resource.load(resource_dict)
assert resource.remote_data_path is None
def test_push_datapackage(storage):
# Prepare and call
descriptor = helpers.fixture_path('datapackage', 'datapackage.json')
storage.buckets = ['data___data'] # Without patch it's a reflection
module.push_datapackage(descriptor=descriptor, backend='backend')
# Assert mocked calls
storage.create.assert_called_with(['data___data'], [{
'fields': [{
'name': 'id',
'type': 'integer'
}, {
'name': 'city',
'type': 'string'
}]
}])
storage.write.assert_called_with('data___data', ANY)
# Assert writen data
data = storage.write.call_args[0][1]
assert list(data) == [
(1, 'London'),
(2, 'Paris'),
]
def test_load_accepts_absolute_paths(self):
path = test_helpers.fixture_path('foo.txt')
resource_dict = {
'path': path,
}
resource = datapackage.Resource.load(resource_dict)
assert resource.data == b'foo\n'
def test_load_accepts_absolute_paths(self):
path = test_helpers.fixture_path('foo.txt')
resource_dict = {
'path': path,
}
resource = datapackage.Resource.load(resource_dict)
assert resource.data == b'foo\n'
def test_load_binary_data(self):
resource_dict = {
'path': test_helpers.fixture_path('image.gif'),
}
resource = datapackage.Resource.load(resource_dict)
with open(resource_dict['path'], 'rb') as f:
assert resource.data == f.read()
def test_local_data_path(self, datapackage_zip):
dp = datapackage.DataPackage(datapackage_zip)
assert dp.resources[0].local_data_path is not None
with open(test_helpers.fixture_path('foo.txt')) as data_file:
with open(dp.resources[0].local_data_path) as local_data_file:
assert local_data_file.read() == data_file.read()
def test_with_local_resources_with_existent_path_isnt_safe(self):
descriptor = {
'resources': [
{'path': test_helpers.fixture_path('foo.txt')},
]
}
with pytest.raises(datapackage.exceptions.DataPackageException):
dp = datapackage.DataPackage(descriptor, {})
def test_with_local_resources_with_existent_path_isnt_safe(self):
metadata = {
'resources': [
{'path': test_helpers.fixture_path('foo.txt')},
]
}
dp = datapackage.DataPackage(metadata, {})
assert not dp.safe()
def test_local_data_path(self, datapackage_zip):
dp = datapackage.DataPackage(datapackage_zip)
assert dp.resources[0].local_data_path is not None
with open(test_helpers.fixture_path('foo.txt')) as data_file:
with open(dp.resources[0].local_data_path) as local_data_file:
assert local_data_file.read() == data_file.read()
def test_works_with_resources_with_relative_paths(self, tmpfile):
path = test_helpers.fixture_path(
'datapackage_with_foo.txt_resource.json'
)
dp = datapackage.DataPackage(path)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
assert len(z.filelist) == 2
def test_load_prefers_loading_local_data_over_url(self):
httpretty.HTTPretty.allow_net_connect = False
resource_dict = {
'path': test_helpers.fixture_path('foo.txt'),
'url': 'http://someplace.com/inexistent-file.txt',
}
resource = datapackage.Resource.load(resource_dict)
assert resource.data == b'foo\n'
def test_load_prefers_loading_local_data_over_url(self):
httpretty.HTTPretty.allow_net_connect = False
resource_dict = {
'path': test_helpers.fixture_path('foo.txt'),
'url': 'http://someplace.com/inexistent-file.txt',
}
resource = datapackage.Resource.load(resource_dict)
assert resource.data == b'foo\n'
def _create_resource_file_with(self, fixture):
path = test_helpers.fixture_path(fixture)
with open(path, 'rb') as f:
body = f.read()
url = 'http://www.someplace.com/{fixture}'.format(fixture=fixture)
httpretty.register_uri(httpretty.GET, url, body=body)
return RemoteResourceFile(url)
def test_load_binary_data(self):
resource_dict = {
'path': test_helpers.fixture_path('image.gif'),
}
resource = datapackage.Resource.load(resource_dict)
with open(resource_dict['path'], 'rb') as f:
assert resource.data == f.read()
def test_load_accepts_relative_paths(self):
filename = 'foo.txt'
base_path = os.path.dirname(test_helpers.fixture_path(filename))
resource_dict = {
'path': filename,
}
resource = datapackage.Resource.load(resource_dict, base_path)
assert resource.data == b'foo\n'
def test_local_data_path_returns_the_absolute_path(self):
base_path = test_helpers.fixture_path('')
path = os.path.join(base_path, '..', 'fixtures', 'unicode.txt')
resource_dict = {
'path': path,
}
resource = datapackage.Resource.load(resource_dict)
abs_path = os.path.join(base_path, 'unicode.txt')
assert resource.local_data_path == abs_path
def test_local_data_path_returns_the_absolute_path_relative_to_base(self):
base_path = test_helpers.fixture_path('')
resource_dict = {
'path': 'unicode.txt',
'base': base_path,
}
resource = datapackage.Resource.load(resource_dict)
abs_path = os.path.join(base_path, resource_dict['path'])
assert resource.local_data_path == abs_path
def test_local_data_path_returns_the_absolute_path(self):
base_path = test_helpers.fixture_path('')
path = os.path.join(base_path, '..', 'fixtures', 'unicode.txt')
resource_dict = {
'path': path,
}
resource = datapackage.Resource.load(resource_dict)
abs_path = os.path.join(base_path, 'unicode.txt')
assert resource.local_data_path == abs_path
def _create_resource_file_with(self, fixture):
path = test_helpers.fixture_path(fixture)
with open(path, 'rb') as f:
body = f.read()
url = 'http://www.someplace.com/{fixture}'.format(fixture=fixture)
httpretty.register_uri(httpretty.GET,
url,
body=body)
return RemoteResourceFile(url)
def test_load_accepts_relative_paths(self):
filename = 'foo.txt'
base_path = os.path.dirname(
test_helpers.fixture_path(filename)
)
resource_dict = {
'path': filename,
}
resource = datapackage.Resource.load(resource_dict, base_path)
assert resource.data == b'foo\n'
def datapackage_zip(tmpfile):
metadata = {
'name': 'proverbs',
'resources': [
{'path': test_helpers.fixture_path('foo.txt')},
]
}
dp = datapackage.DataPackage(metadata)
dp.save(tmpfile)
return tmpfile
def test_local_resource_with_absolute_path_is_loaded(self):
path = test_helpers.fixture_path('foo.txt')
metadata = {
'resources': [
{'path': path},
],
}
dp = datapackage.DataPackage(metadata)
assert len(dp.resources) == 1
assert dp.resources[0].data == b'foo\n'
def test_with_local_resources_with_existent_path_isnt_safe(self):
descriptor = {
'resources': [
{
'path': test_helpers.fixture_path('foo.txt')
},
]
}
dp = datapackage.DataPackage(descriptor, {})
assert not dp.safe()
def test_local_resource_with_absolute_path_is_loaded(self):
path = test_helpers.fixture_path('foo.txt')
metadata = {
'resources': [
{'path': path},
],
}
dp = datapackage.DataPackage(metadata)
assert len(dp.resources) == 1
assert dp.resources[0].data == b'foo\n'
def datapackage_zip():
with tempfile.NamedTemporaryFile() as f:
metadata = {
'name': 'proverbs',
'resources': [
{'path': test_helpers.fixture_path('foo.txt')},
]
}
dp = datapackage.DataPackage(metadata)
dp.save(f)
yield f
def test_load_accepts_relative_paths_with_base_defined_in_metadata(self):
filename = 'foo.txt'
base_path = os.path.dirname(
test_helpers.fixture_path(filename)
)
resource_dict = {
'path': filename,
'base': base_path,
}
resource = datapackage.Resource.load(resource_dict)
assert resource.data == b'foo\n'
def datapackage_zip(tmpfile):
metadata = {
'name': 'proverbs',
'resources': [
{
'path': test_helpers.fixture_path('foo.txt')
},
]
}
dp = datapackage.DataPackage(metadata)
dp.save(tmpfile)
return tmpfile
def test_load_base_path_in_metadata_overloads_base_passed_in_args(self):
filename = 'foo.txt'
base_path = os.path.dirname(
test_helpers.fixture_path(filename)
)
resource_dict = {
'path': filename,
'base': base_path,
}
resource = datapackage.Resource.load(resource_dict,
'invalid_base_path')
assert resource.data == b'foo\n'
def test_works_with_resources_with_relative_paths(self, tmpfile):
base_path = test_helpers.fixture_path('')
metadata = {
'name': 'proverbs',
'base': base_path,
'resources': [
{'path': 'unicode.txt'}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
def test_adds_resources_inside_data_subfolder(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {'name': 'proverbs', 'resources': [{'path': resource_path}]}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
filename = [
name for name in z.namelist() if name.startswith('data/')
]
assert len(filename) == 1
resource_data = z.read(filename[0]).decode('utf-8')
assert resource_data == '万事开头难\n'
def test_generates_filenames_for_named_resources(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {
'name': 'proverbs',
'resources': [
{'name': 'proverbs', 'format': 'TXT', 'path': resource_path},
{'name': 'proverbs_without_format', 'path': resource_path}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
assert 'data/proverbs.txt' in z.namelist()
assert 'data/proverbs_without_format' in z.namelist()
def test_fixes_resources_paths_to_be_relative_to_package(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {
'name': 'proverbs',
'resources': [
{'name': 'unicode', 'format': 'txt', 'path': resource_path}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
json_string = z.read('datapackage.json').decode('utf-8')
generated_dp_dict = json.loads(json_string)
assert generated_dp_dict['resources'][0]['path'] == 'data/unicode.txt'
def test_fixes_resources_paths_to_be_relative_to_package(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {
'name': 'proverbs',
'resources': [
{'name': 'unicode', 'format': 'txt', 'path': resource_path}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
json_string = z.read('datapackage.json').decode('utf-8')
generated_dp_dict = json.loads(json_string)
assert generated_dp_dict['resources'][0]['path'] == 'data/unicode.txt'
def test_generates_filenames_for_named_resources(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {
'name': 'proverbs',
'resources': [
{'name': 'proverbs', 'format': 'TXT', 'path': resource_path},
{'name': 'proverbs_without_format', 'path': resource_path}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
assert 'data/proverbs.txt' in z.namelist()
assert 'data/proverbs_without_format' in z.namelist()
def test_adds_resources_inside_data_subfolder(self, tmpfile):
resource_path = test_helpers.fixture_path('unicode.txt')
metadata = {
'name': 'proverbs',
'resources': [
{'path': resource_path}
]
}
schema = {}
dp = datapackage.DataPackage(metadata, schema)
dp.save(tmpfile)
with zipfile.ZipFile(tmpfile, 'r') as z:
filename = [name for name in z.namelist()
if name.startswith('data/')]
assert len(filename) == 1
resource_data = z.read(filename[0]).decode('utf-8')
assert resource_data == '万事开头难\n'
def test_load_tsv(self):
resource_dict = {
'path': test_helpers.fixture_path('cities.tsv')
}
resource = TabularResource(resource_dict)
assert resource.data == [
{'Area': '1807.92', 'Name': 'Acrelândia', 'Population': '12538', 'State': 'AC'},
{'Area': '186.53', 'Name': 'Boca da Mata', 'Population': '25776', 'State': 'AL'},
{'Area': '242.62', 'Name': 'Capela', 'Population': '17077', 'State': 'AL'},
{'Area': '6709.66', 'Name': 'Tartarugalzinho', 'Population': '12563', 'State': 'AP'},
{'Area': '837.72', 'Name': 'América Dourada', 'Population': None, 'State': 'BA'},
{'Area': '204.79', 'Name': 'Jijoca de Jericoacoara', 'Population': '17002', 'State': 'CE'},
{'Area': '6953.67', 'Name': 'Cavalcante', 'Population': '9392', 'State': 'GO'},
{'Area': '8258.42', 'Name': 'Centro Novo do Maranhão', 'Population': '17622', 'State': 'MA'},
{'Area': '3651.18', 'Name': 'Ped\\ro G\\omes', 'Population': '7967', 'State': 'MS'},
{'Area': '881.06', 'Name': 'Abadia dos Dourados', 'Population': '6704', 'State': 'MG'},
]
def test_load_tsv(self):
resource_dict = {
'path': test_helpers.fixture_path('cities.tsv')
}
resource = TabularResource(resource_dict)
assert resource.data == [
{'Area': '1807.92', 'Name': 'Acrelândia', 'Population': '12538', 'State': 'AC'},
{'Area': '186.53', 'Name': 'Boca da Mata', 'Population': '25776', 'State': 'AL'},
{'Area': '242.62', 'Name': 'Capela', 'Population': '17077', 'State': 'AL'},
{'Area': '6709.66', 'Name': 'Tartarugalzinho', 'Population': '12563', 'State': 'AP'},
{'Area': '837.72', 'Name': 'América Dourada', 'Population': None, 'State': 'BA'},
{'Area': '204.79', 'Name': 'Jijoca de Jericoacoara', 'Population': '17002', 'State': 'CE'},
{'Area': '6953.67', 'Name': 'Cavalcante', 'Population': '9392', 'State': 'GO'},
{'Area': '8258.42', 'Name': 'Centro Novo do Maranhão', 'Population': '17622', 'State': 'MA'},
{'Area': '3651.18', 'Name': 'Ped\\ro G\\omes', 'Population': '7967', 'State': 'MS'},
{'Area': '881.06', 'Name': 'Abadia dos Dourados', 'Population': '6704', 'State': 'MG'},
]
def test_push_datapackage(storage):
# Prepare and call
descriptor = helpers.fixture_path('datapackage', 'datapackage.json')
storage.tables = ['data___data'] # Without patch it's a reflection
module.push_datapackage(descriptor=descriptor, backend='backend')
# Assert mocked calls
storage.create.assert_called_with(
['data___data'],
[{'fields': [
{'name': 'id', 'type': 'integer'},
{'name': 'city', 'type': 'string'}]}])
storage.write.assert_called_with('data___data', ANY)
# Assert writen data
data = storage.write.call_args[0][1]
assert list(data) == [
(1, 'London'),
(2, 'Paris'),
]
def test_local_with_relative_resources_paths_is_safe(self):
fixture_name = 'datapackage_with_foo.txt_resource.json'
path = test_helpers.fixture_path(fixture_name)
dp = datapackage.DataPackage(path, {})
assert dp.safe()
def test_base_path_is_datapackages_base_path_when_it_is_a_file(self):
path = test_helpers.fixture_path('empty_datapackage.json')
base_path = os.path.dirname(path)
dp = datapackage.DataPackage(path)
assert dp.base_path == base_path
def test_init_raises_if_path_json_isnt_a_dict(self):
empty_array_path = test_helpers.fixture_path('empty_array.json')
with pytest.raises(datapackage.exceptions.DataPackageException):
datapackage.DataPackage(empty_array_path)
def test_init_raises_if_path_isnt_a_json(self):
not_a_json_path = test_helpers.fixture_path('not_a_json')
with pytest.raises(datapackage_validate.exceptions.SchemaError):
Schema(not_a_json_path)
def test_init_raises_if_path_isnt_a_json(self):
not_a_json_path = test_helpers.fixture_path('not_a_json')
with pytest.raises(datapackage.exceptions.DataPackageException):
datapackage.DataPackage(not_a_json_path)
def test_init_raises_if_path_isnt_a_json(self):
not_a_json_path = test_helpers.fixture_path('not_a_json')
with pytest.raises(datapackage.exceptions.DataPackageException):
datapackage.DataPackage(not_a_json_path)
def test_init_raises_if_path_json_isnt_a_dict(self):
empty_array_path = test_helpers.fixture_path('empty_array.json')
with pytest.raises(datapackage.exceptions.DataPackageException):
datapackage.DataPackage(empty_array_path)
def test_init_loads_schema_from_path(self):
schema_path = test_helpers.fixture_path('empty_schema.json')
assert Schema(schema_path).to_dict() == {}
def test_local_with_relative_resources_paths_is_safe(self):
fixture_name = 'datapackage_with_foo.txt_resource.json'
path = test_helpers.fixture_path(fixture_name)
dp = datapackage.DataPackage(path, {})
assert dp.safe()
def test_init_accepts_file_paths(self):
path = test_helpers.fixture_path('empty_datapackage.json')
dp = datapackage.DataPackage(path)
assert dp.metadata == {}
def test_raises_if_profile_in_local_file_isnt_a_json(self):
registry_path = test_helpers.fixture_path('registry_with_notajson_profile.csv')
registry = datapackage_registry.Registry(registry_path)
with assert_raises(DataPackageRegistryException):
registry.get('notajson')
def test_local_data_path_returns_the_unmodified_path(self):
resource_dict = {
'path': test_helpers.fixture_path('unicode.txt'),
}
resource = datapackage.Resource.load(resource_dict)
assert resource.local_data_path == resource_dict['path']
def test_local_resource_with_relative_path_is_loaded(self):
datapackage_filename = 'datapackage_with_foo.txt_resource.json'
path = test_helpers.fixture_path(datapackage_filename)
dp = datapackage.DataPackage(path)
assert len(dp.resources) == 1
assert dp.resources[0].data == b'foo\n'