def test_create_files_in_json_and_xlsx_format(self):
node = create_node('sample_data.json')
file_generator = CatalogFileGenerator(node)
file_generator.generate_files()
json_file_path = os.path.join(settings.MEDIA_ROOT, CATALOG_ROOT,
node.catalog_id, 'data.json')
xlsx_file_path = os.path.join(settings.MEDIA_ROOT, CATALOG_ROOT,
node.catalog_id, 'catalog.xlsx')
self.assertTrue(os.path.exists(json_file_path))
self.assertTrue(os.path.exists(xlsx_file_path))
def test_json_file_is_not_served_as_attachment(self):
node = create_node('another_catalog.json')
CatalogFileGenerator(node).generate_files()
response = self.client.get(reverse("django_datajsonar:json_catalog",
args=['test_catalog']))
content_disposition = response["Content-Disposition"]
self.assertNotIn("attachment;", content_disposition)
def test_json_file_content_in_response_content(self):
node = create_node('another_catalog.json')
CatalogFileGenerator(node).generate_files()
response = self.client.get(reverse("django_datajsonar:json_catalog",
args=['test_catalog']))
response_json = json.loads(response.getvalue().decode('utf-8'))
with open_catalog('another_catalog.json') as file:
file_json = json.loads(file.read().decode('utf-8'))
self.assertEquals(response_json, file_json)
def index(self, node, task):
self._reset_catalog_if_exists(node)
try:
catalog = DataJson(node.catalog_url,
catalog_format=node.catalog_format,
verify_ssl=self.indexing_config.verify_ssl)
catalog.generate_distribution_ids()
node.catalog = json.dumps(catalog)
node.save()
except NonParseableCatalog as e:
self._set_catalog_as_errored(node)
ReadDataJsonTask.info(task, READ_ERROR.format(node.catalog_id, e))
return
self.reset_fields(node)
self._index_catalog(catalog, node, task)
file_generator = CatalogFileGenerator(node)
file_generator.generate_files()
def test_catalog_url_request_does_not_verify_ssl_by_default(self, _database_loader):
node = Node.objects.create(catalog_id='test_catalog',
catalog_format='json',
catalog_url='https://fakeurl.com/data.json',
indexable=True)
with open_catalog('sample_data.json') as sample:
text = sample.read()
with requests_mock.Mocker() as m:
m.get('https://fakeurl.com/data.json', status_code=200, content=text)
CatalogFileGenerator(node).generate_files()
self.assertEqual(2, len(m.request_history))
for request in m.request_history:
self.assertFalse(request.verify)
def test_creates_new_xlsx_file_if_node_format_is_json(self, mock):
node = Node.objects.create(catalog_id='test_catalog',
catalog_url='https://fakeurl.com/data.json',
catalog_format='json',
indexable=True)
with open_catalog('sample_data.json') as sample:
text = sample.read()
with requests_mock.Mocker() as m:
m.get('https://fakeurl.com/data.json',
status_code=200,
content=text)
CatalogFileGenerator(node).generate_files()
mock.assert_called_once()
def test_created_files_are_saved_in_node_model(self):
node = create_node('sample_data.json')
CatalogFileGenerator(node).generate_files()
json_file_path = os.path.join(settings.MEDIA_ROOT, CATALOG_ROOT,
node.catalog_id, 'data.json')
xlsx_file_path = os.path.join(settings.MEDIA_ROOT, CATALOG_ROOT,
node.catalog_id, 'catalog.xlsx')
with open(json_file_path, 'rb') as json_sample:
json_file_content = json_sample.read()
with open(xlsx_file_path, 'rb') as xlsx_sample:
xlsx_file_content = xlsx_sample.read()
node_json_file_content = node.json_catalog_file.read()
node_xlsx_file_content = node.xlsx_catalog_file.read()
self.assertEquals(json_file_content, node_json_file_content)
self.assertEquals(xlsx_file_content, node_xlsx_file_content)
def test_creates_datajson_with_arguments_specified_in_node(
self, datajson_mock):
node = Node.objects.create(
catalog_id='test_catalog',
catalog_format='json',
catalog_url='https://fakeurl.com/without_format',
indexable=True,
verify_ssl=True)
with open_catalog('sample_data.json') as sample:
text = sample.read()
with requests_mock.Mocker() as m:
m.get('https://fakeurl.com/without_format',
status_code=200,
content=text)
CatalogFileGenerator(node).generate_files()
datajson_mock.assert_called_once_with(node.catalog_url,
catalog_format='json',
verify_ssl=True)