def test_same_day_multiple_catalog_uploads(node):
with open_catalog('data.json') as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
CatalogUpload.create_from_url_or_file(data_dict)
with open_catalog('data.json') as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
CatalogUpload.create_from_url_or_file(data_dict)
assert CatalogUpload.objects.count() == 1
def test_raises_validation_error_if_catalog_is_not_valid(node):
with open_catalog('catalogo-justicia.xlsx') as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'xlsx', 'node': node, 'file': temp_file}
validator = CatalogDataValidator()
with pytest.raises(ValidationError):
validator.get_and_validate_data(data_dict)
def test_upload_distribution_filename_without_extension(distribution):
distribution.file_name = 'file_without_extension'
with freeze_time('2019-01-01'):
with open_catalog('test_data.csv') as f:
distribution.distributionupload_set.create(file=File(f))
dist_filename = Path(DistributionUpload.objects.first().file.name).name
assert dist_filename == 'file_without_extension-2019-01-01'
def test_create_from_file(node):
filename = 'data.json'
with open_catalog(filename) as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
catalog = CatalogUpload.create_from_url_or_file(data_dict)
assert b'dataset' in catalog.json_file.read()
def test_returns_correct_data_when_uploading_file(node):
with open_catalog('data.json') as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
validator = CatalogDataValidator()
data = validator.get_and_validate_data(data_dict)
assert b'dataset' in data['json_file'].read()
def test_file_upload_large_name(distribution):
long_name = 'extremely_long_distribution_id_that_makes_final_name_very_big'
distribution.identifier = long_name
distribution.save()
with open_catalog('test_data.csv') as f:
distribution.distributionupload_set.create(file=File(f))
assert DistributionUpload.objects.count() == 1
def distribution_upload():
_catalog(
) # Necesario que exista un catálogo para crear un distribution upload exitorsamente
with open_catalog('test_data.csv') as distribution_fd:
model = _distribution().distributionupload_set.create(
file=File(distribution_fd))
return model
def test_catalog_upload_creates_both_formats(node):
filename = 'data.json'
with open_catalog(filename) as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
catalog = CatalogUpload.create_from_url_or_file(data_dict)
assert catalog.json_file is not None
assert catalog.xlsx_file is not None
def _catalog():
with open_catalog('data.json') as catalog_fd:
model = CatalogUpload(format=CatalogUpload.FORMAT_JSON,
json_file=File(catalog_fd),
node=_node())
model.save()
return model
def xlsx_catalog():
with open_catalog('catalogo-justicia_valido.xlsx') as catalog_fd:
model = CatalogUpload(format=CatalogUpload.FORMAT_XLSX,
xlsx_file=File(catalog_fd),
node=_node())
model.save()
return model
def test_older_versions_listed(logged_client, distribution_upload):
distribution = distribution_upload.distribution
with freeze_time('2019-01-01'):
with open_catalog('test_data.csv') as fd:
other = distribution.distributionupload_set \
.create(file=File(fd))
response = _call(logged_client, distribution)
assert str(other.uploaded_at) in response.content.decode('utf-8')
def test_create_from_file(admin_client, catalog):
with open_catalog('test_data.csv') as sample:
form_data = {'file': sample,
'dataset_identifier': "125",
'distribution_identifier': "125.1",
'file_name': 'test_data.csv'}
admin_client.post(_add_url(catalog.node), form_data)
assert DistributionUpload.objects.get().distribution.identifier == "125.1"
def _create_dist2(distribution, distribution_upload):
last_week = datetime.datetime.now() - datetime.timedelta(days=7)
DistributionUpload.objects.filter(pk=distribution_upload.pk).update(
uploaded_at=last_week)
with open_catalog('test_data.csv') as distribution_fd:
dist2 = distribution.distributionupload_set.create(
file=File(distribution_fd))
dist2.save()
return dist2
def test_file_saved_as_latest(distribution):
with open_catalog('test_data.csv') as f:
distribution.distributionupload_set.create(file=File(f))
assert os.path.exists(
os.path.join(settings.MEDIA_ROOT, 'catalog',
distribution.catalog.identifier, 'dataset', '125',
'distribution', '125.1', 'download',
distribution.file_name))
def test_redirects_even_if_catalog_is_not_valid(self):
with open_catalog('data.json') as sample:
form_data = {
'format': 'json',
'node': self.node.identifier,
'file': sample
}
response = self.client.post(
reverse('catalog:add_catalog',
kwargs={'node_id': self.node.id}), form_data)
self.assertEqual(response.status_code, 302)
def test_catalog_is_created_when_submitted_form_is_valid(self):
with open_catalog('valid_data.json') as sample:
form_data = {
'format': 'json',
'node': self.node.identifier,
'file': sample
}
self.client.post(
reverse('catalog:add_catalog',
kwargs={'node_id': self.node.id}), form_data)
self.assertEqual(1, CatalogUpload.objects.count())
def test_create_from_valid_url(node, requests_mock):
with open_catalog('data.json') as sample:
requests_mock.get("https://datos.gob.ar/data.json",
content=sample.read())
data_dict = {
'format': 'json',
'node': node,
'url': "https://datos.gob.ar/data.json"
}
catalog = CatalogUpload.create_from_url_or_file(data_dict)
assert catalog is not None
def test_context_manager_does_not_lose_files_using_same_file_name(client, distribution_upload):
file_path = join('tests_media', distribution_upload.file_path())
file_path_with_date = join('tests_media', distribution_upload.file_path(with_date=True))
with open_catalog('test_data.csv') as sample:
raw_data = {'node': distribution_upload.distribution.catalog,
'dataset_identifier': distribution_upload.distribution.dataset_identifier,
'distribution_identifier': distribution_upload.distribution.identifier,
'file_name': distribution_upload.distribution.file_name,
'file': sample}
client.post(_add_url(distribution_upload.distribution.catalog), raw_data)
assert isfile(file_path) and isfile(file_path_with_date)
def test_error_messages_in_view_if_catalog_is_not_valid(self):
with open_catalog('data.json') as sample:
form_data = {
'format': 'json',
'node_id': self.node.id,
'file': sample
}
response = self.client.post(reverse(
'catalog:add_catalog', kwargs={'node_id': self.node.id}),
form_data,
follow=True)
self.assertIsNotNone(response.context['messages'])
def test_posting_new_version_twice_persists_only_one_instance(client, catalog):
with open_catalog('test_data.csv') as sample:
form_data = {'file': sample,
'dataset_identifier': "125",
'distribution_identifier': "125.1",
'file_name': 'test_data.csv'}
client.post(_add_url(catalog.node), form_data)
sample.seek(0)
client.post(_add_version_url(catalog.node, form_data['distribution_identifier']), form_data)
assert DistributionUpload.objects.count() == 1
def test_returns_correct_data_when_specifying_url(node, requests_mock):
with open_catalog('data.json') as sample:
text = sample.read()
requests_mock.get("https://datos.gob.ar/data.json", content=text)
data_dict = {
'format': 'json',
'node': node,
'url': 'https://datos.gob.ar/data.json'
}
validator = CatalogDataValidator()
data = validator.get_and_validate_data(data_dict)
assert data['json_file'].read() == text
def test_generated_file_paths_for_distribution(admin_client, catalog):
with open_catalog('test_data.csv') as sample:
form_data = {'file': sample,
'dataset_identifier': "125",
'distribution_identifier': "125.1",
'file_name': 'test_data.csv'}
admin_client.post(_add_url(catalog.node), form_data)
distribution = DistributionUpload.objects.get()
assert str(distribution.file_path()) == 'catalog/test_id/dataset/125/distribution/125.1/' \
'download/test_data.csv'
assert str(distribution.file_path(with_date=True)) == f'catalog/test_id/dataset/125/' \
f'distribution/125.1/download/test_data' \
f'-{timezone.now().date()}.csv'
def test_returns_correct_data_when_specifying_url_with_explicit_format(
node, requests_mock):
with open_catalog('xlsx_catalog.xlsx') as sample:
text = sample.read()
requests_mock.get('https://fakeurl.com/export?format=xlsx',
content=text)
data_dict = {
'format': 'xlsx',
'node': node,
'url': 'https://fakeurl.com/export?format=xlsx'
}
validator = CatalogDataValidator()
data = validator.get_and_validate_data(data_dict)
assert data['xlsx_file'].read() == text
def test_created_catalog_has_both_file_formats(self):
with open_catalog('valid_data.json') as sample:
form_data = {
'format': 'json',
'node': self.node.identifier,
'file': sample
}
self.client.post(
reverse('catalog:add_catalog',
kwargs={'node_id': self.node.id}), form_data)
self.assertEqual(1, CatalogUpload.objects.count())
catalog = CatalogUpload.objects.first()
self.assertIsNotNone(catalog.json_file)
self.assertIsNotNone(catalog.xlsx_file)
def test_new_version_form_contains_previous_data(client, catalog):
with open_catalog('test_data.csv') as sample:
form_data = {'file': sample,
'dataset_identifier': "125",
'distribution_identifier': "an_easily_findable_distribution_identifier",
'file_name': 'an_easily_findable_file_name.csv'}
client.post(_add_url(catalog.node), form_data)
sample.seek(0)
response = client.get(_add_version_url(catalog.node, form_data['distribution_identifier']))
response_content = response.content.decode('utf-8')
assert "125" in response_content
assert "an_easily_findable_distribution_identifier" in response_content
assert "an_easily_findable_file_name.csv" in response_content
def test_context_manager_removes_old_same_day_version_file_if_name_changes(client,
distribution_upload):
old_file_path = join('tests_media', distribution_upload.file_path())
with open_catalog('test_data.csv') as sample:
raw_data = {'node': distribution_upload.distribution.catalog,
'dataset_identifier': distribution_upload.distribution.dataset_identifier,
'distribution_identifier': distribution_upload.distribution.identifier,
'file_name': 'new_file_name.csv',
'file': sample}
client.post(_add_url(distribution_upload.distribution.catalog), raw_data)
updated_distribution = DistributionUpload.objects \
.get(distribution=distribution_upload.distribution)
new_file_path = join(settings.MEDIA_ROOT, updated_distribution.file_path())
assert new_file_path != old_file_path
assert not isfile(old_file_path)
assert isfile(new_file_path)
def test_catalog_history_has_object_list_ordered_by_date_descendent(
admin_client, catalog):
lastweek = datetime.datetime.now() - datetime.timedelta(days=7)
CatalogUpload.objects.filter(pk=catalog.pk).update(uploaded_at=lastweek)
with open_catalog('data.json') as catalog_fd:
model = CatalogUpload(format=CatalogUpload.FORMAT_JSON,
json_file=File(catalog_fd),
node=catalog.node)
model.save()
response = admin_client.get(
reverse('catalog:catalog_history', kwargs={'node_id':
catalog.node.id}))
catalogs = response.context['object_list']
for i in range(len(catalogs) - 1):
assert catalogs[i].uploaded_at > catalogs[i + 1].uploaded_at
def test_form_submit_with_valid_data_redirects_to_success_page(self):
with open_catalog('valid_data.json') as sample:
form_data = {
'format': 'json',
'node': self.node.identifier,
'file': sample
}
response = self.client.post(
reverse('catalog:add_catalog',
kwargs={'node_id': self.node.id}), form_data)
self.assertEqual(response.status_code, 302)
self.assertEqual(
response.url,
reverse('catalog:upload_success',
kwargs={'node_id': self.node.id}))
def test_validate_returns_error_message_if_catalog_is_not_valid(node):
error_messages = [
"'publisher' is a required property",
"'title' is a required property",
"'superThemeTaxonomy' is a required property",
"'description' is a required property",
"'Índice-precios-internos-basicos-al-por-mayor-desagregado-base-1993-anual.csv' "
"is not valid under any of the given schemas",
]
with open_catalog('data.json') as sample:
temp_file = temp_uploaded_file(sample)
data_dict = {'format': 'json', 'node': node, 'file': temp_file}
catalog_upload = CatalogUpload.create_from_url_or_file(data_dict)
validation_result = catalog_upload.validate()
for error_message in error_messages:
assert error_message in validation_result
def test_view_messages_includes_error_messages_from_validator(self):
error_messages = [
"'title' is a required property",
"'description' is a required property",
"'publisher' is a required property",
"'superThemeTaxonomy' is a required property",
"'Índice-precios-internos-basicos-al-por-mayor-desagregado-base-1993-anual.csv' "
"is not valid under any of the given schemas",
]
with open_catalog('data.json') as sample:
form_data = {
'format': 'json',
'node': self.node.identifier,
'file': sample
}
response = self.client.post(reverse(
'catalog:add_catalog', kwargs={'node_id': self.node.id}),
form_data,
follow=True)
messages = [
str(message) for message in list(response.context['messages'])
]
self.assertCountEqual(error_messages, messages)
