def test_purge_orphan_community():
dataset = Dataset.objects.create(title='test_dataset')
community_resource1 = CommunityResourceFactory(title='test_community_1')
community_resource2 = CommunityResourceFactory(title='test_community_2')
community_resource1.dataset = dataset
community_resource1.save()
tasks.purge_orphan_community_resources()
assert CommunityResource.objects.count() == 1
def test_community_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
community_resource1 = CommunityResourceFactory()
community_resource1.dataset = dataset
community_resource1.save()
assert len(dataset.community_resources) == 1
community_resource2 = CommunityResourceFactory()
community_resource2.dataset = dataset
community_resource2.save()
assert len(dataset.community_resources) == 2
assert dataset.community_resources[1].id == community_resource1.id
assert dataset.community_resources[0].id == community_resource2.id
def test_my_org_community_resources(self):
user = self.login()
member = Member(user=user, role='editor')
organization = OrganizationFactory(members=[member])
community_resources = [
CommunityResourceFactory(owner=user) for _ in range(2)]
org_community_resources = [
CommunityResourceFactory(organization=organization)
for _ in range(2)]
response = self.get(url_for('api.my_org_community_resources'))
self.assert200(response)
self.assertEqual(
len(response.json),
len(community_resources) + len(org_community_resources))
def test_community_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
community_resource1 = CommunityResourceFactory()
community_resource1.dataset = dataset
community_resource1.save()
self.assertEqual(len(dataset.community_resources), 1)
community_resource2 = CommunityResourceFactory()
community_resource2.dataset = dataset
community_resource2.save()
self.assertEqual(len(dataset.community_resources), 2)
self.assertEqual(dataset.community_resources[1].id,
community_resource1.id)
self.assertEqual(dataset.community_resources[0].id,
community_resource2.id)
def test_get_specific(self):
'''Should fetch serialized resource from the API based on rid'''
resources = [ResourceFactory() for _ in range(7)]
specific_resource = ResourceFactory(
id='817204ac-2202-8b4a-98e7-4284d154d10c', title='my-resource')
resources.append(specific_resource)
dataset = DatasetFactory(resources=resources)
response = self.get(url_for('apiv2.resource',
rid=specific_resource.id))
self.assert200(response)
data = response.json
assert data['dataset_id'] == str(dataset.id)
assert data['resource']['id'] == str(specific_resource.id)
assert data['resource']['title'] == specific_resource.title
response = self.get(
url_for('apiv2.resource',
rid='111111ac-1111-1b1a-11e1-1111d111d11c'))
self.assert404(response)
com_resource = CommunityResourceFactory()
response = self.get(url_for('apiv2.resource', rid=com_resource.id))
self.assert200(response)
data = response.json
assert data['dataset_id'] is None
assert data['resource']['id'] == str(com_resource.id)
assert data['resource']['title'] == com_resource.title
def test_community_resource_latest_url(self):
'''It should redirect to the real community resource URL'''
resource = CommunityResourceFactory()
response = self.get(url_for('datasets.resource',
id=resource.id))
self.assertStatus(response, 302)
self.assertEqual(response.location, resource.url)
def test_purge_datasets_community():
dataset = Dataset.objects.create(title='delete me', deleted='2016-01-01')
community_resource1 = CommunityResourceFactory()
community_resource1.dataset = dataset
community_resource1.save()
tasks.purge_datasets()
assert CommunityResource.objects.count() == 0
def test_community_resource_api_get(self):
'''It should fetch a community resource from the API'''
with self.autoindex():
community_resource = CommunityResourceFactory()
response = self.get(
url_for('api.community_resource', community=community_resource))
self.assert200(response)
data = json.loads(response.data)
self.assertEqual(data['id'], str(community_resource.id))
def test_my_org_community_resources_with_search(self):
user = self.login()
member = Member(user=user, role='editor')
organization = OrganizationFactory(members=[member])
community_resources = [
CommunityResourceFactory(owner=user, title='foô'),
]
org_community_resources = [
CommunityResourceFactory(organization=organization, title='foô'),
]
# Should not be listed.
CommunityResourceFactory(owner=user)
CommunityResourceFactory(organization=organization)
response = self.get(url_for('api.my_org_community_resources'),
qs={'q': 'foô'})
self.assert200(response)
self.assertEqual(
len(response.json),
len(community_resources) + len(org_community_resources))
def test_community_resource_api_update_w_previous_owner(self):
'''Should update a community resource and keep the original author'''
owner = UserFactory()
community_resource = CommunityResourceFactory(owner=owner)
self.login(AdminFactory())
data = community_resource.to_dict()
data['description'] = 'new description'
response = self.put(
url_for('api.community_resource', community=community_resource),
data)
self.assert200(response)
self.assertEqual(CommunityResource.objects.first().owner, owner)
def test_community_resource_api_update(self):
'''It should update a community resource from the API'''
user = self.login()
community_resource = CommunityResourceFactory(owner=user)
data = community_resource.to_dict()
data['description'] = 'new description'
response = self.put(
url_for('api.community_resource', community=community_resource),
data)
self.assert200(response)
self.assertEqual(CommunityResource.objects.count(), 1)
self.assertEqual(CommunityResource.objects.first().description,
'new description')
def test_no_preview_for_community_resources():
domain = faker.domain_name()
remote_id = faker.unique_string()
dataset = DatasetFactory(
extras={
'harvest:remote_id': remote_id,
'harvest:domain': domain,
'ods:url': faker.uri(),
})
resource = CommunityResourceFactory(dataset=dataset,
extras={'ods:type': 'api'})
assert resource.preview_url is None
def generate_fixtures(source):
'''Build sample fixture data (users, datasets, reuses) from local or remote file.'''
if source.startswith('http'):
json_fixtures = requests.get(source).json()
else:
with open(source) as f:
json_fixtures = json.load(f)
with click.progressbar(json_fixtures) as bar:
for fixture in bar:
user = UserFactory()
if not fixture['organization']:
dataset = DatasetFactory(**fixture['dataset'], owner=user)
else:
org = Organization.objects(
id=fixture['organization']['id']).first()
if not org:
org = OrganizationFactory(**fixture['organization'],
members=[Member(user=user)])
dataset = DatasetFactory(**fixture['dataset'],
organization=org)
for resource in fixture['resources']:
res = ResourceFactory(**resource)
dataset.add_resource(res)
for reuse in fixture['reuses']:
ReuseFactory(**reuse, datasets=[dataset], owner=user)
for community in fixture['community_resources']:
CommunityResourceFactory(**community,
dataset=dataset,
owner=user)
for discussion in fixture['discussions']:
messages = discussion.pop('discussion')
DiscussionFactory(**discussion,
subject=dataset,
user=user,
discussion=[
MessageDiscussionFactory(**message,
posted_by=user)
for message in messages
])
def test_community_resource_api_update_with_file(self):
'''It should update a community resource file from the API'''
dataset = VisibleDatasetFactory()
user = self.login()
community_resource = CommunityResourceFactory(dataset=dataset,
owner=user)
response = self.post(url_for('api.upload_community_resource',
community=community_resource),
{'file': (StringIO(b'aaa'), 'test.txt')},
json=False)
self.assert200(response)
data = json.loads(response.data)
self.assertEqual(data['id'], str(community_resource.id))
self.assertEqual(data['title'], 'test.txt')
data['description'] = 'new description'
response = self.put(
url_for('api.community_resource', community=community_resource),
data)
self.assert200(response)
self.assertEqual(CommunityResource.objects.count(), 1)
self.assertEqual(CommunityResource.objects.first().description,
'new description')
self.assertTrue(
CommunityResource.objects.first().url.endswith('test.txt'))
def test_delete(self):
'''It should delete the connected user'''
user = self.login()
self.assertIsNone(user.deleted)
other_user = UserFactory()
members = [Member(user=user), Member(user=other_user)]
organization = OrganizationFactory(members=members)
disc_msg_content = faker.sentence()
disc_msg = DiscMsg(content=disc_msg_content, posted_by=user)
other_disc_msg_content = faker.sentence()
other_disc_msg = DiscMsg(content=other_disc_msg_content,
posted_by=other_user)
discussion = DiscussionFactory(user=user,
discussion=[disc_msg, other_disc_msg])
issue_msg_content = faker.sentence()
issue_msg = IssueMsg(content=issue_msg_content, posted_by=user)
other_issue_msg_content = faker.sentence()
other_issue_msg = IssueMsg(content=other_issue_msg_content,
posted_by=other_user)
issue = IssueFactory(user=user,
discussion=[issue_msg, other_issue_msg])
dataset = DatasetFactory(owner=user)
reuse = ReuseFactory(owner=user)
resource = CommunityResourceFactory(owner=user)
activity = UserCreatedDataset.objects().create(actor=user,
related_to=dataset)
following = Follow.objects().create(follower=user,
following=other_user)
followed = Follow.objects().create(follower=other_user, following=user)
with capture_mails() as mails:
response = self.delete(url_for('api.me'))
self.assertEqual(len(mails), 1)
self.assertEqual(mails[0].send_to, set([user.email]))
self.assertEqual(mails[0].subject, _('Account deletion'))
self.assert204(response)
user.reload()
organization.reload()
discussion.reload()
issue.reload()
dataset.reload()
reuse.reload()
resource.reload()
activity.reload()
# The following are deleted
with self.assertRaises(Follow.DoesNotExist):
following.reload()
# The followers are deleted
with self.assertRaises(Follow.DoesNotExist):
followed.reload()
# The personal data of the user are anonymized
self.assertEqual(user.email, '{}@deleted'.format(user.id))
self.assertEqual(user.password, None)
self.assertEqual(user.active, False)
self.assertEqual(user.first_name, 'DELETED')
self.assertEqual(user.last_name, 'DELETED')
self.assertFalse(bool(user.avatar))
self.assertEqual(user.avatar_url, None)
self.assertEqual(user.website, None)
self.assertEqual(user.about, None)
# The user is marked as deleted
self.assertIsNotNone(user.deleted)
# The user is removed from his organizations
self.assertEqual(len(organization.members), 1)
self.assertEqual(organization.members[0].user.id, other_user.id)
# The discussions are kept but the messages are anonymized
self.assertEqual(len(discussion.discussion), 2)
self.assertEqual(discussion.discussion[0].content, 'DELETED')
self.assertEqual(discussion.discussion[1].content,
other_disc_msg_content)
# The issues are kept and the messages are not anonymized
self.assertEqual(len(issue.discussion), 2)
self.assertEqual(issue.discussion[0].content, issue_msg_content)
self.assertEqual(issue.discussion[1].content, other_issue_msg_content)
# The datasets are unchanged
self.assertEqual(dataset.owner, user)
# The reuses are unchanged
self.assertEqual(reuse.owner, user)
# The community resources are unchanged
self.assertEqual(resource.owner, user)
# The activities are unchanged
self.assertEqual(activity.actor, user)
def test_community_resource_deleted_dataset(self):
dataset = DatasetFactory()
community_resource = CommunityResourceFactory(dataset=dataset)
community_resource.dataset.delete()
community_resource.reload()
assert community_resource.dataset is None
def test_json_ld(self):
'''It should render a json-ld markup into the dataset page'''
resource = ResourceFactory(format='png',
description='* Title 1\n* Title 2',
metrics={'views': 10})
license = LicenseFactory(url='http://www.datagouv.fr/licence')
dataset = DatasetFactory(license=license,
tags=['foo', 'bar'],
resources=[resource],
description='a&éèëù$£',
owner=UserFactory(),
extras={'foo': 'bar'})
community_resource = CommunityResourceFactory(
dataset=dataset,
format='csv',
description='* Title 1\n* Title 2',
metrics={'views': 42})
url = url_for('datasets.show', dataset=dataset)
response = self.get(url)
self.assert200(response)
json_ld = self.get_json_ld(response)
self.assertEquals(json_ld['@context'], 'http://schema.org')
self.assertEquals(json_ld['@type'], 'Dataset')
self.assertEquals(json_ld['@id'], str(dataset.id))
self.assertEquals(json_ld['description'], 'a&éèëù$£')
self.assertEquals(json_ld['alternateName'], dataset.slug)
self.assertEquals(json_ld['dateCreated'][:16],
dataset.created_at.isoformat()[:16])
self.assertEquals(json_ld['dateModified'][:16],
dataset.last_modified.isoformat()[:16])
self.assertEquals(json_ld['url'], 'http://localhost{}'.format(url))
self.assertEquals(json_ld['name'], dataset.title)
self.assertEquals(json_ld['keywords'], 'bar,foo')
self.assertEquals(len(json_ld['distribution']), 1)
json_ld_resource = json_ld['distribution'][0]
self.assertEquals(json_ld_resource['@type'], 'DataDownload')
self.assertEquals(json_ld_resource['@id'], str(resource.id))
self.assertEquals(json_ld_resource['url'], resource.latest)
self.assertEquals(json_ld_resource['name'], resource.title)
self.assertEquals(json_ld_resource['contentUrl'], resource.url)
self.assertEquals(json_ld_resource['dateCreated'][:16],
resource.created_at.isoformat()[:16])
self.assertEquals(json_ld_resource['dateModified'][:16],
resource.modified.isoformat()[:16])
self.assertEquals(json_ld_resource['datePublished'][:16],
resource.published.isoformat()[:16])
self.assertEquals(json_ld_resource['encodingFormat'], 'png')
self.assertEquals(json_ld_resource['contentSize'],
resource.filesize)
self.assertEquals(json_ld_resource['fileFormat'], resource.mime)
self.assertEquals(json_ld_resource['description'],
'Title 1 Title 2')
self.assertEquals(json_ld_resource['interactionStatistic'],
{
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': 10,
})
self.assertEquals(len(json_ld['contributedDistribution']), 1)
json_ld_resource = json_ld['contributedDistribution'][0]
self.assertEquals(json_ld_resource['@type'], 'DataDownload')
self.assertEquals(json_ld_resource['@id'], str(community_resource.id))
self.assertEquals(json_ld_resource['url'], community_resource.latest)
self.assertEquals(json_ld_resource['name'], community_resource.title)
self.assertEquals(json_ld_resource['contentUrl'],
community_resource.url)
self.assertEquals(json_ld_resource['dateCreated'][:16],
community_resource.created_at.isoformat()[:16])
self.assertEquals(json_ld_resource['dateModified'][:16],
community_resource.modified.isoformat()[:16])
self.assertEquals(json_ld_resource['datePublished'][:16],
community_resource.published.isoformat()[:16])
self.assertEquals(json_ld_resource['encodingFormat'],
community_resource.format)
self.assertEquals(json_ld_resource['contentSize'],
community_resource.filesize)
self.assertEquals(json_ld_resource['fileFormat'],
community_resource.mime)
self.assertEquals(json_ld_resource['description'], 'Title 1 Title 2')
self.assertEquals(json_ld_resource['interactionStatistic'], {
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': 42,
})
self.assertEquals(json_ld['extras'],
[{
'@type': 'http://schema.org/PropertyValue',
'name': 'foo',
'value': 'bar',
}])
self.assertEquals(json_ld['license'], 'http://www.datagouv.fr/licence')
self.assertEquals(json_ld['author']['@type'], 'Person')
def community_resource(app):
community_resource = CommunityResourceFactory()
download(community_resource)
download(community_resource, latest=True)
return community_resource