def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras["harvest:domain"] = "test.org"
dataset.extras["harvest:remote_id"] = str(i)
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({"local": str(dataset.id), "remote": str(index)})
csvfile.flush()
result = actions.attach("test.org", csvfile.name)
dbcount = Dataset.objects(**{"extras__harvest:remote_id__exists": True}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras["harvest:domain"], "test.org")
self.assertEqual(dataset.extras["harvest:remote_id"], str(index))
def test_datasets_csv(self):
with self.autoindex():
datasets = [DatasetFactory(resources=[ResourceFactory()])
for _ in range(5)]
hidden_dataset = DatasetFactory()
response = self.get(url_for('site.datasets_csv'))
self.assert200(response)
self.assertEqual(response.mimetype, 'text/csv')
self.assertEqual(response.charset, 'utf-8')
csvfile = StringIO.StringIO(response.data)
reader = csv.get_reader(csvfile)
header = reader.next()
self.assertEqual(header[0], 'id')
self.assertIn('title', header)
self.assertIn('description', header)
self.assertIn('created_at', header)
self.assertIn('last_modified', header)
self.assertIn('tags', header)
self.assertIn('metric.reuses', header)
rows = list(reader)
ids = [row[0] for row in rows]
self.assertEqual(len(rows), len(datasets))
for dataset in datasets:
self.assertIn(str(dataset.id), ids)
self.assertNotIn(str(hidden_dataset.id), ids)
def test_render_home(self):
'''It should render the home page'''
for i in range(3):
org = OrganizationFactory()
DatasetFactory(organization=org)
ReuseFactory(organization=org)
current_site.settings.home_datasets = [
DatasetFactory() for _ in range(3)]
current_site.settings.home_reuses = [
ReuseFactory() for _ in range(3)]
response = self.get(url_for('site.home'))
self.assert200(response)
def test_attach(self):
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
self.assertEqual(result.success, len(datasets))
self.assertEqual(result.errors, 0)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def test_attach(self):
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
self.assertEqual(result.success, len(datasets))
self.assertEqual(result.errors, 0)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def test_attach_skip_not_found(self):
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
writer.writerow({
'local': 'not-found',
'remote': '42'
})
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
self.assertEqual(result.success, len(datasets))
self.assertEqual(result.errors, 1)
def test_resources_csv_with_filters(self):
'''Should handle filtering but ignore paging or facets'''
with self.autoindex():
filtered_datasets = [
DatasetFactory(
resources=[ResourceFactory(),
ResourceFactory()],
tags=['selected']) for _ in range(6)
]
[DatasetFactory(resources=[ResourceFactory()]) for _ in range(3)]
DatasetFactory()
response = self.get(
url_for('site.resources_csv',
tag='selected',
page_size=3,
facets=True))
self.assert200(response)
self.assertEqual(response.mimetype, 'text/csv')
self.assertEqual(response.charset, 'utf-8')
csvfile = StringIO.StringIO(response.data)
reader = csv.get_reader(csvfile)
header = reader.next()
self.assertEqual(header[0], 'dataset.id')
self.assertIn('dataset.title', header)
self.assertIn('dataset.url', header)
self.assertIn('title', header)
self.assertIn('description', header)
self.assertIn('type', header)
self.assertIn('url', header)
self.assertIn('created_at', header)
self.assertIn('modified', header)
self.assertIn('downloads', header)
resource_id_index = header.index('id')
rows = list(reader)
ids = [(row[0], row[resource_id_index]) for row in rows]
self.assertEqual(len(rows),
sum(len(d.resources) for d in filtered_datasets))
for dataset in filtered_datasets:
for resource in dataset.resources:
self.assertIn((str(dataset.id), str(resource.id)), ids)
def test_render_metrics(self):
'''It should render the search page'''
for i in range(3):
org = OrganizationFactory()
DatasetFactory(organization=org)
ReuseFactory(organization=org)
response = self.get(url_for('site.dashboard'))
self.assert200(response)
def test_render_reuse_page(self):
'''It should render the reuse page'''
org = OrganizationFactory()
dataset = DatasetFactory(organization=org)
reuse = ReuseFactory(organization=org, datasets=[dataset])
response = self.get(url_for('reuses.show', reuse=reuse))
self.assert200(response)
def test_render_dataset_page(self):
'''It should render the dataset page'''
org = OrganizationFactory()
dataset = DatasetFactory(organization=org)
ReuseFactory(organization=org, datasets=[dataset])
response = self.get(url_for('datasets.show', dataset=dataset))
self.assert200(response)
def test_datasets_csv_with_filters(self):
'''Should handle filtering but ignore paging or facets'''
with self.autoindex():
filtered_datasets = [
DatasetFactory(resources=[ResourceFactory()],
tags=['selected']) for _ in range(6)
]
datasets = [
DatasetFactory(resources=[ResourceFactory()]) for _ in range(3)
]
hidden_dataset = DatasetFactory()
response = self.get(
url_for('site.datasets_csv',
tag='selected',
page_size=3,
facets=True))
self.assert200(response)
self.assertEqual(response.mimetype, 'text/csv')
self.assertEqual(response.charset, 'utf-8')
csvfile = StringIO.StringIO(response.data)
reader = csv.get_reader(csvfile)
header = reader.next()
self.assertEqual(header[0], 'id')
self.assertIn('title', header)
self.assertIn('description', header)
self.assertIn('created_at', header)
self.assertIn('last_modified', header)
self.assertIn('tags', header)
self.assertIn('metric.reuses', header)
rows = list(reader)
ids = [row[0] for row in rows]
# Should ignore paging
self.assertEqual(len(rows), len(filtered_datasets))
# SHoulf pass filter
for dataset in filtered_datasets:
self.assertIn(str(dataset.id), ids)
for dataset in datasets:
self.assertNotIn(str(dataset.id), ids)
self.assertNotIn(str(hidden_dataset.id), ids)
def test_render_organization_page(self):
'''It should render the organization page'''
org = OrganizationFactory()
datasets = [DatasetFactory(organization=org) for _ in range(3)]
for dataset in datasets:
ReuseFactory(organization=org, datasets=[dataset])
response = self.get(url_for('organizations.show', org=org))
self.assert200(response)
def test_render_home(self):
'''It should render the home page'''
with self.autoindex():
for i in range(3):
org = OrganizationFactory()
DatasetFactory(organization=org)
ReuseFactory(organization=org)
response = self.get(url_for('site.home'))
self.assert200(response)
def test_render_search(self):
'''It should render the search page'''
with self.autoindex():
for i in range(3):
org = OrganizationFactory()
DatasetFactory(organization=org)
ReuseFactory(organization=org)
response = self.get(url_for('front.search'))
self.assert200(response)
def test_resources_csv(self):
with self.autoindex():
datasets = [
DatasetFactory(
resources=[ResourceFactory(),
ResourceFactory()]) for _ in range(3)
]
hidden_dataset = DatasetFactory()
response = self.get(url_for('site.resources_csv'))
self.assert200(response)
self.assertEqual(response.mimetype, 'text/csv')
self.assertEqual(response.charset, 'utf-8')
csvfile = StringIO.StringIO(response.data)
reader = csv.get_reader(csvfile)
header = reader.next()
self.assertEqual(header[0], 'dataset.id')
self.assertIn('dataset.title', header)
self.assertIn('dataset.url', header)
self.assertIn('title', header)
self.assertIn('description', header)
self.assertIn('type', header)
self.assertIn('url', header)
self.assertIn('created_at', header)
self.assertIn('modified', header)
self.assertIn('downloads', header)
resource_id_index = header.index('id')
rows = list(reader)
ids = [(row[0], row[resource_id_index]) for row in rows]
self.assertEqual(len(rows), sum(len(d.resources) for d in datasets))
for dataset in datasets:
for resource in dataset.resources:
self.assertIn((str(dataset.id), str(resource.id)), ids)
def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras['harvest:domain'] = 'test.org'
dataset.extras['harvest:remote_id'] = str(i)
dataset.last_modified = datetime.now()
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
dbcount = Dataset.objects(**{
'extras__harvest:remote_id__exists': True
}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def test_attach_does_not_duplicate(self):
attached_datasets = []
for i in range(2):
dataset = DatasetFactory.build()
dataset.extras['harvest:domain'] = 'test.org'
dataset.extras['harvest:remote_id'] = str(i)
dataset.last_modified = datetime.now()
dataset.save()
attached_datasets.append(dataset)
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
dbcount = Dataset.objects(**{
'extras__harvest:remote_id__exists': True
}).count()
self.assertEqual(result.success, len(datasets))
self.assertEqual(dbcount, result.success)
for index, dataset in enumerate(datasets):
dataset.reload()
self.assertEqual(dataset.extras['harvest:domain'], 'test.org')
self.assertEqual(dataset.extras['harvest:remote_id'], str(index))
def test_attach_skip_not_found(self):
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=["local", "remote"], delimiter=b";", quotechar=b'"')
writer.writeheader()
writer.writerow({"local": "not-found", "remote": "42"})
for index, dataset in enumerate(datasets):
writer.writerow({"local": str(dataset.id), "remote": str(index)})
csvfile.flush()
result = actions.attach("test.org", csvfile.name)
self.assertEqual(result.success, len(datasets))
self.assertEqual(result.errors, 1)
def test_attach_skip_not_found(self):
datasets = DatasetFactory.create_batch(3)
with NamedTemporaryFile() as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['local', 'remote'],
delimiter=b';',
quotechar=b'"')
writer.writeheader()
writer.writerow({'local': 'not-found', 'remote': '42'})
for index, dataset in enumerate(datasets):
writer.writerow({
'local': str(dataset.id),
'remote': str(index)
})
csvfile.flush()
result = actions.attach('test.org', csvfile.name)
self.assertEqual(result.success, len(datasets))
self.assertEqual(result.errors, 1)
def test_redirect_datasets(self):
dataset = DatasetFactory()
response = self.client.get('/en/dataset/%s/' % dataset.slug)
self.assertRedirects(
response, url_for('datasets.show', dataset=dataset))
def process(self, item):
mock_process.send(self, item=item)
return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))
def process(self, item):
mock_process.send(self, item=item)
return DatasetFactory.build(title='dataset-{0}'.format(item.remote_id))