def test_publish_message_resource_created(self):
kafka_mock = Mock()
KafkaProducerSingleton.get_instance = lambda: kafka_mock
dataset = DatasetFactory()
resource = ResourceFactory()
expected_signals = (Dataset.on_resource_added, )
with assert_emit(*expected_signals):
dataset.add_resource(resource)
producer = KafkaProducerSingleton.get_instance()
message_type = f'resource.{KafkaMessageType.CREATED.value}'
expected_value = {
'service': 'udata',
'data': serialize_resource_for_event(resource),
'meta': {
'message_type': message_type,
'dataset_id': str(dataset.id)
}
}
topic = f"{current_app.config['UDATA_INSTANCE_NAME']}.{message_type}"
producer.send.assert_called_with(topic,
value=expected_value,
key=str(resource.id).encode("utf-8"))
def test_add_resource_missing_checksum_type(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
resource.checksum.type = None
with self.assertRaises(db.ValidationError):
dataset.add_resource(resource)
def test_add_resource_missing_checksum_type(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
resource.checksum.type = None
with pytest.raises(db.ValidationError):
dataset.add_resource(resource)
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
self.assertEqual(len(dataset.resources), 1)
with assert_emit(*expected_signals):
dataset.add_resource(resource)
self.assertEqual(len(dataset.resources), 2)
self.assertEqual(dataset.resources[0].id, resource.id)
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
def test_add_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
expected_signals = (Dataset.on_resource_added, )
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory())
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].dataset == dataset
def test_add_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
expected_signals = (post_save, Dataset.after_save, Dataset.on_update,
Dataset.on_resource_added)
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory())
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].dataset == dataset
def test_quality_all(self):
user = UserFactory()
visitor = UserFactory()
dataset = DatasetFactory(owner=user,
frequency='weekly',
tags=['foo', 'bar'],
description='a' * 42)
dataset.add_resource(ResourceFactory(format='pdf'))
DiscussionFactory(
subject=dataset,
user=visitor,
discussion=[MessageDiscussionFactory(posted_by=visitor)])
self.assertEqual(dataset.quality['score'], 0)
self.assertEqual(sorted(dataset.quality.keys()), [
'description_length', 'discussions', 'frequency',
'has_only_closed_or_no_formats', 'has_resources',
'has_unavailable_resources', 'has_untreated_discussions', 'score',
'tags_count', 'update_in'
])
def generate_fixtures(source):
'''Build sample fixture data (users, datasets, reuses) from local or remote file.'''
if source.startswith('http'):
json_fixtures = requests.get(source).json()
else:
with open(source) as f:
json_fixtures = json.load(f)
with click.progressbar(json_fixtures) as bar:
for fixture in bar:
user = UserFactory()
if not fixture['organization']:
dataset = DatasetFactory(**fixture['dataset'], owner=user)
else:
org = Organization.objects(
id=fixture['organization']['id']).first()
if not org:
org = OrganizationFactory(**fixture['organization'],
members=[Member(user=user)])
dataset = DatasetFactory(**fixture['dataset'],
organization=org)
for resource in fixture['resources']:
res = ResourceFactory(**resource)
dataset.add_resource(res)
for reuse in fixture['reuses']:
ReuseFactory(**reuse, datasets=[dataset], owner=user)
for community in fixture['community_resources']:
CommunityResourceFactory(**community,
dataset=dataset,
owner=user)
for discussion in fixture['discussions']:
messages = discussion.pop('discussion')
DiscussionFactory(**discussion,
subject=dataset,
user=user,
discussion=[
MessageDiscussionFactory(**message,
posted_by=user)
for message in messages
])
def test_quality_all(self):
user = UserFactory()
visitor = UserFactory()
dataset = DatasetFactory(owner=user, frequency='weekly',
tags=['foo', 'bar'], description='a' * 42)
dataset.add_resource(ResourceFactory(format='pdf'))
DiscussionFactory(
subject=dataset, user=visitor,
discussion=[MessageDiscussionFactory(posted_by=visitor)])
assert dataset.quality['score'] == 0
assert sorted(dataset.quality.keys()) == [
'description_length',
'discussions',
'frequency',
'has_only_closed_or_no_formats',
'has_resources',
'has_unavailable_resources',
'has_untreated_discussions',
'score',
'tags_count',
'update_in'
]
def test_last_update_with_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
dataset.add_resource(resource)
self.assertEqualDates(dataset.last_update, resource.published)
def test_quality_has_undefined_and_closed_format(self):
dataset = DatasetFactory(description='', )
dataset.add_resource(ResourceFactory(format=None))
dataset.add_resource(ResourceFactory(format='xls'))
assert dataset.quality['has_only_closed_or_no_formats']
assert dataset.quality['score'] == 0
def test_last_update_with_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
dataset.add_resource(resource)
assert_equal_dates(dataset.last_update, resource.published)
def test_quality_has_undefined_and_closed_format(self):
dataset = DatasetFactory(description='', )
dataset.add_resource(ResourceFactory(format=None))
dataset.add_resource(ResourceFactory(format='xls'))
self.assertTrue(dataset.quality['has_only_closed_or_no_formats'])
self.assertEqual(dataset.quality['score'], 0)
def test_quality_has_opened_formats(self):
dataset = DatasetFactory(description='', )
dataset.add_resource(ResourceFactory(format='pdf'))
dataset.add_resource(ResourceFactory(format='csv'))
self.assertFalse(dataset.quality['has_only_closed_or_no_formats'])
self.assertEqual(dataset.quality['score'], 4)
def test_quality_has_opened_formats(self):
dataset = DatasetFactory(description='', )
dataset.add_resource(ResourceFactory(format='pdf'))
dataset.add_resource(ResourceFactory(format='csv'))
assert not dataset.quality['has_only_closed_or_no_formats']
assert dataset.quality['score'] == 4
def test_quality_has_only_closed_formats(self):
dataset = DatasetFactory(description='', )
dataset.add_resource(ResourceFactory(format='pdf'))
assert dataset.quality['has_only_closed_or_no_formats']
assert dataset.quality['score'] == 0
