def test_reuse_metrics(self):
dataset = VisibleDatasetFactory()
reuse = VisibleReuseFactory()
issue = IssueFactory(subject=reuse)
DiscussionFactory(subject=reuse)
reuse.count_datasets()
reuse.count_issues()
reuse.count_discussions()
assert reuse.get_metrics()['datasets'] == 1
assert reuse.get_metrics()['issues'] == 1
assert reuse.get_metrics()['discussions'] == 1
with assert_emit(Reuse.on_update):
reuse.datasets.append(dataset)
reuse.save()
reuse.count_datasets()
assert reuse.get_metrics()['datasets'] == 2
dataset.count_reuses()
assert dataset.get_metrics()['reuses'] == 1
with assert_emit(Reuse.on_update):
reuse.datasets.remove(dataset)
reuse.save()
dataset_tasks.update_datasets_reuses_metrics()
dataset.reload()
assert dataset.get_metrics()['reuses'] == 0
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
def test_add_resource_without_checksum(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory(checksum=None)
expected_signals = post_save, Dataset.after_save, Dataset.on_update
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory(checksum=None))
self.assertEqual(len(dataset.resources), 1)
with assert_emit(*expected_signals):
dataset.add_resource(resource)
self.assertEqual(len(dataset.resources), 2)
self.assertEqual(dataset.resources[0].id, resource.id)
def test_add_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
expected_signals = (Dataset.on_resource_added, )
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory())
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].dataset == dataset
def test_default(self):
org = OrganizationFactory()
source = HarvestSourceFactory(backend='factory', organization=org)
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert len(job.items) == COUNT
for item in job.items:
assert item.status == 'done'
assert item.errors == []
assert item.started is not None
assert item.ended is not None
assert item.dataset is not None
dataset = item.dataset
assert Dataset.objects(id=dataset.id).first() is not None
assert dataset.organization == org
assert 'harvest:remote_id' in dataset.extras
assert 'harvest:last_update' in dataset.extras
assert 'harvest:source_id' in dataset.extras
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == COUNT
def test_default(self):
org = OrganizationFactory()
source = HarvestSourceFactory(backend='factory', organization=org)
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert len(job.items) == COUNT
for item in job.items:
assert item.status == 'done'
assert item.errors == []
assert item.started is not None
assert item.ended is not None
assert item.dataset is not None
dataset = item.dataset
assert Dataset.objects(id=dataset.id).first() is not None
assert dataset.organization == org
assert 'harvest:remote_id' in dataset.extras
assert 'harvest:last_update' in dataset.extras
assert 'harvest:source_id' in dataset.extras
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == COUNT
def test_ignore_post_save_signal(self):
dataset = DatasetFactory()
unexpected_signals = Dataset.after_save, Dataset.on_update
with assert_not_emit(*unexpected_signals), assert_emit(post_save):
dataset.title = 'New title'
dataset.save(signal_kwargs={'ignores': ['post_save']})
def test_add_resource(self):
user = UserFactory()
dataset = DatasetFactory(owner=user)
resource = ResourceFactory()
expected_signals = (post_save, Dataset.after_save, Dataset.on_update,
Dataset.on_resource_added)
with assert_emit(*expected_signals):
dataset.add_resource(ResourceFactory())
assert len(dataset.resources) == 1
with assert_emit(*expected_signals):
dataset.add_resource(resource)
assert len(dataset.resources) == 2
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].dataset == dataset
def test_default(self):
org = OrganizationFactory()
source = HarvestSourceFactory(backend='factory', organization=org)
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
self.assertEqual(len(HarvestJob.objects(source=source)), 1)
job = source.get_last_job()
self.assertEqual(job.status, 'done')
self.assertEqual(job.errors, [])
self.assertIsNotNone(job.started)
self.assertIsNotNone(job.ended)
self.assertEqual(len(job.items), COUNT)
for item in job.items:
self.assertEqual(item.status, 'done')
self.assertEqual(item.errors, [])
self.assertIsNotNone(item.started)
self.assertIsNotNone(item.ended)
self.assertIsNotNone(item.dataset)
dataset = item.dataset
self.assertIsNotNone(Dataset.objects(id=dataset.id).first())
self.assertEqual(dataset.organization, org)
self.assertIn('harvest:remote_id', dataset.extras)
self.assertIn('harvest:last_update', dataset.extras)
self.assertIn('harvest:source_id', dataset.extras)
self.assertEqual(len(HarvestJob.objects), 1)
self.assertEqual(len(Dataset.objects), COUNT)
def test_publish_message_resource_removed(self):
kafka_mock = Mock()
KafkaProducerSingleton.get_instance = lambda: kafka_mock
resource = ResourceFactory()
dataset = DatasetFactory(resources=[resource])
expected_signals = (Dataset.on_resource_removed, )
with assert_emit(*expected_signals):
dataset.remove_resource(resource)
producer = KafkaProducerSingleton.get_instance()
message_type = f'resource.{KafkaMessageType.DELETED.value}'
expected_value = {
'service': 'udata',
'data': None,
'meta': {
'message_type': message_type,
'dataset_id': str(dataset.id)
}
}
topic = f"{current_app.config['UDATA_INSTANCE_NAME']}.{message_type}"
producer.send.assert_called_with(topic,
value=expected_value,
key=str(resource.id).encode("utf-8"))
def test_ignore_post_save_signal(self):
dataset = DatasetFactory()
unexpected_signals = Dataset.after_save, Dataset.on_update
with assert_not_emit(*unexpected_signals), assert_emit(post_save):
dataset.title = 'New title'
dataset.save(signal_kwargs={'ignores': ['post_save']})
def test_ignore_post_save_signal(self):
resource = ResourceFactory()
DatasetFactory(resources=[resource])
unexpected_signals = Dataset.after_save, Dataset.on_update
with assert_not_emit(*unexpected_signals), assert_emit(post_save):
resource.title = 'New title'
resource.save(signal_kwargs={'ignores': ['post_save']})
def test_create(self):
data = self.factory.as_dict()
url = url_for('api.organization_badges', org=self.organization)
with self.api_user(), assert_emit(on_badge_added):
response = self.post(url, data)
self.assert201(response)
self.organization.reload()
self.assertEqual(len(self.organization.badges), 1)
def test_create(self, api):
data = self.factory.as_dict()
url = url_for('api.organization_badges', org=self.organization)
with assert_emit(on_badge_added):
response = api.post(url, data)
assert201(response)
self.organization.reload()
assert len(self.organization.badges) is 1
def test_ignore_post_save_signal(self):
resource = ResourceFactory()
DatasetFactory(resources=[resource])
unexpected_signals = Dataset.after_save, Dataset.on_update
with assert_not_emit(*unexpected_signals), assert_emit(post_save):
resource.title = 'New title'
resource.save(signal_kwargs={'ignores': ['post_save']})
def test_ignore_post_save_signal(self):
resource = ResourceFactory()
# assigning to a variable to avoid garbage collection issue
_ = DatasetFactory(resources=[resource])
unexpected_signals = Dataset.after_save, Dataset.on_update
with assert_not_emit(*unexpected_signals), assert_emit(post_save):
resource.title = 'New title'
resource.save(signal_kwargs={'ignores': ['post_save']})
def test_delete_source_by_objectid(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_deleted):
deleted_source = actions.delete_source(source.id)
self.assertIsNotNone(deleted_source.deleted)
self.assertEqual(deleted_source.id, source.id)
deleted_sources = HarvestSource.objects(deleted__exists=True)
self.assertEqual(len(deleted_sources), 1)
def test_delete_source_by_objectid(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_deleted):
deleted_source = actions.delete_source(source.id)
assert deleted_source.deleted is not None
assert deleted_source.id == source.id
deleted_sources = HarvestSource.objects(deleted__exists=True)
assert len(deleted_sources) == 1
def test_delete_source_by_objectid(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_deleted):
deleted_source = actions.delete_source(source.id)
assert deleted_source.deleted is not None
assert deleted_source.id == source.id
deleted_sources = HarvestSource.objects(deleted__exists=True)
assert len(deleted_sources) == 1
def test_need_update(self):
'''It should update the metric on "need_update" signal'''
obj = FakeModel.objects.create()
metric = FakeMetric(obj)
with assert_emit(FakeMetric.need_update, FakeMetric.updated):
metric.trigger_update()
obj.reload()
assert obj.metrics['fake'] == 42
def test_need_update(self):
'''It should update the metric on "need_update" signal'''
obj = FakeModel.objects.create()
metric = FakeMetric(obj)
with assert_emit(FakeMetric.need_update, FakeMetric.updated):
metric.trigger_update()
obj.reload()
assert obj.metrics['fake'] == 42
def test_reschedule(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), hour=0)
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), minute=0)
self.assertEqual(len(PeriodicTask.objects), 1)
periodic_task = source.periodic_task
self.assertEqual(periodic_task, PeriodicTask.objects.first())
self.assertEqual(periodic_task.args, [str(source.id)])
self.assertEqual(periodic_task.crontab.hour, '*')
self.assertEqual(periodic_task.crontab.minute, '0')
self.assertEqual(periodic_task.crontab.day_of_week, '*')
self.assertEqual(periodic_task.crontab.day_of_month, '*')
self.assertEqual(periodic_task.crontab.month_of_year, '*')
self.assertTrue(periodic_task.enabled)
self.assertEqual(periodic_task.name, 'Harvest {0}'.format(source.name))
def test_reschedule(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), hour=0)
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), minute=0)
assert len(PeriodicTask.objects) == 1
periodic_task = source.periodic_task
assert periodic_task == PeriodicTask.objects.first()
assert periodic_task.args == [str(source.id)]
assert periodic_task.crontab.hour == '*'
assert periodic_task.crontab.minute == '0'
assert periodic_task.crontab.day_of_week == '*'
assert periodic_task.crontab.day_of_month == '*'
assert periodic_task.crontab.month_of_year == '*'
assert periodic_task.enabled
assert periodic_task.name == 'Harvest {0}'.format(source.name)
def test_reschedule(self):
source = HarvestSourceFactory()
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), hour=0)
with assert_emit(signals.harvest_source_scheduled):
source = actions.schedule(str(source.id), minute=0)
assert len(PeriodicTask.objects) == 1
periodic_task = source.periodic_task
assert periodic_task == PeriodicTask.objects.first()
assert periodic_task.args == [str(source.id)]
assert periodic_task.crontab.hour == '*'
assert periodic_task.crontab.minute == '0'
assert periodic_task.crontab.day_of_week == '*'
assert periodic_task.crontab.day_of_month == '*'
assert periodic_task.crontab.month_of_year == '*'
assert periodic_task.enabled
assert periodic_task.name == 'Harvest {0}'.format(source.name)
def test_update(self, app):
'''It should store the updated metric on "updated" signal'''
with assert_emit(FakeSiteMetric.need_update, FakeSiteMetric.updated):
FakeSiteMetric.update()
metrics = Metrics.objects.last_for(app.config['SITE_ID'])
assert metrics.values['fake'] == FAKE_VALUE
site = Site.objects.get(id=app.config['SITE_ID'])
assert site.metrics['fake'] == FAKE_VALUE
def test_create_same(self):
data = self.factory.as_dict()
url = url_for('api.organization_badges', org=self.organization)
with self.api_user():
with assert_emit(on_badge_added):
self.post(url, data)
with assert_not_emit(on_badge_added):
response = self.post(url, data)
self.assertStatus(response, 200)
self.organization.reload()
self.assertEqual(len(self.organization.badges), 1)
def test_create_source_with_config(self):
source_url = faker.url()
config = {'filters': [{'key': 'test', 'value': 42}]}
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory',
config=config)
assert source.config == config
def test_update_source(self):
source = HarvestSourceFactory()
data = source.to_dict()
new_url = faker.url()
data['url'] = new_url
with assert_emit(signals.harvest_source_updated):
source = actions.update_source(source.id, data)
assert source.url == new_url
source.reload()
assert source.url == new_url
def test_updated(self):
'''It should store the updated metric on "updated" signal'''
obj = FakeModel.objects.create()
metric = FakeMetric(obj)
metric.value = 'some-value'
with assert_emit(FakeMetric.updated):
with assert_not_emit(FakeMetric.need_update):
metric.notify_update()
metrics = Metrics.objects.last_for(obj)
assert metrics.values['fake'] == 'some-value'
def test_update_source(self):
source = HarvestSourceFactory()
data = source.to_dict()
new_url = faker.url()
data['url'] = new_url
with assert_emit(signals.harvest_source_updated):
source = actions.update_source(source.id, data)
self.assertEqual(source.url, new_url)
source.reload()
self.assertEqual(source.url, new_url)
def test_delete(self, api):
badge = self.factory()
self.organization.add_badge(badge.kind)
self.organization.save()
url = url_for('api.organization_badge',
org=self.organization,
badge_kind=str(badge.kind))
with assert_emit(on_badge_removed):
response = api.delete(url)
assert204(response)
self.organization.reload()
assert len(self.organization.badges) is 0
def test_updated(self):
'''It should store the updated metric on "updated" signal'''
obj = FakeModel.objects.create()
metric = FakeMetric(obj)
metric.value = 'some-value'
with assert_emit(FakeMetric.updated):
with assert_not_emit(FakeMetric.need_update):
metric.notify_update()
metrics = Metrics.objects.last_for(obj)
assert metrics.values['fake'] == 'some-value'
def test_update_resource(self):
user = UserFactory()
resource = ResourceFactory()
dataset = DatasetFactory(owner=user, resources=[resource])
expected_signals = post_save, Dataset.after_save, Dataset.on_update
resource.description = 'New description'
with assert_emit(*expected_signals):
dataset.update_resource(resource)
self.assertEqual(len(dataset.resources), 1)
self.assertEqual(dataset.resources[0].id, resource.id)
self.assertEqual(dataset.resources[0].description, 'New description')
def test_update_resource(self):
user = UserFactory()
resource = ResourceFactory()
dataset = DatasetFactory(owner=user, resources=[resource])
expected_signals = post_save, Dataset.after_save, Dataset.on_update
resource.description = 'New description'
with assert_emit(*expected_signals):
dataset.update_resource(resource)
assert len(dataset.resources) == 1
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].description == 'New description'
def test_update_resource(self):
user = UserFactory()
resource = ResourceFactory()
dataset = DatasetFactory(owner=user, resources=[resource])
expected_signals = (Dataset.on_resource_updated, )
resource.description = 'New description'
with assert_emit(*expected_signals):
dataset.update_resource(resource)
assert len(dataset.resources) == 1
assert dataset.resources[0].id == resource.id
assert dataset.resources[0].description == 'New description'
def test_delete(self):
badge = self.factory()
self.organization.badges.append(badge)
self.organization.save()
url = url_for('api.organization_badge',
org=self.organization,
badge_kind=str(badge.kind))
with self.api_user():
with assert_emit(on_badge_removed):
response = self.delete(url)
self.assertStatus(response, 204)
self.organization.reload()
self.assertEqual(len(self.organization.badges), 0)
def test_unschedule(self):
periodic_task = PeriodicTask.objects.create(
task='harvest',
name=faker.name(),
description=faker.sentence(),
enabled=True,
crontab=PeriodicTask.Crontab())
source = HarvestSourceFactory(periodic_task=periodic_task)
with assert_emit(signals.harvest_source_unscheduled):
actions.unschedule(str(source.id))
source.reload()
self.assertEqual(len(PeriodicTask.objects), 0)
self.assertIsNone(source.periodic_task)
def test_create_source_with_config(self):
source_url = faker.url()
config = {
'filters': [{'key': 'test', 'value': 42}],
'features': {'key': True},
}
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory',
config=config)
assert source.config == config
def test_unschedule(self):
periodic_task = PeriodicTask.objects.create(
task='harvest',
name=faker.name(),
description=faker.sentence(),
enabled=True,
crontab=PeriodicTask.Crontab()
)
source = HarvestSourceFactory(periodic_task=periodic_task)
with assert_emit(signals.harvest_source_unscheduled):
actions.unschedule(str(source.id))
source.reload()
assert len(PeriodicTask.objects) == 0
assert source.periodic_task is None
def test_empty(self):
source = HarvestSourceFactory(backend='factory', config={'count': 0})
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
self.assertEqual(len(HarvestJob.objects(source=source)), 1)
job = source.get_last_job()
self.assertEqual(job.status, 'done')
self.assertEqual(job.errors, [])
self.assertIsNotNone(job.started)
self.assertIsNotNone(job.ended)
self.assertEqual(job.items, [])
self.assertEqual(len(HarvestJob.objects), 1)
self.assertEqual(len(Dataset.objects), 0)
def test_empty(self):
source = HarvestSourceFactory(backend='factory', config={'count': 0})
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert job.items == []
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == 0
def test_empty(self):
source = HarvestSourceFactory(backend='factory', config={'count': 0})
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'done'
assert job.errors == []
assert job.started is not None
assert job.ended is not None
assert job.items == []
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == 0
def test_create_source(self):
source_url = faker.url()
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source', source_url,
'factory')
self.assertEqual(source.name, 'Test source')
self.assertEqual(source.slug, 'test-source')
self.assertEqual(source.url, source_url)
self.assertEqual(source.backend, 'factory')
self.assertEqual(source.frequency, 'manual')
self.assertTrue(source.active)
self.assertIsNone(source.owner)
self.assertIsNone(source.organization)
self.assertEqual(source.validation.state, VALIDATION_PENDING)
self.assertIsNone(source.validation.on)
self.assertIsNone(source.validation.by)
self.assertIsNone(source.validation.comment)
def test_error_on_item(self):
def process(self, item):
if item.remote_id == '1':
raise ValueError('test')
source = HarvestSourceFactory(backend='factory')
with assert_emit(signals.before_harvest_job,
signals.after_harvest_job), \
mock_process.connected_to(process):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'done-errors'
assert job.started is not None
assert job.ended is not None
assert len(job.errors) == 0
assert len(job.items) == COUNT
items_ok = filter(lambda i: not len(i.errors), job.items)
assert len(items_ok) == COUNT - 1
for item in items_ok:
assert item.started is not None
assert item.ended is not None
assert item.status == 'done'
assert item.errors == []
item_ko = filter(lambda i: len(i.errors), job.items)[0]
assert item_ko.started is not None
assert item_ko.ended is not None
assert item_ko.status == 'failed'
assert len(item_ko.errors) == 1
error = item_ko.errors[0]
assert isinstance(error, HarvestError)
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == COUNT - 1
def test_create_source(self):
source_url = faker.url()
with assert_emit(signals.harvest_source_created):
source = actions.create_source('Test source',
source_url,
'factory')
assert source.name == 'Test source'
assert source.slug == 'test-source'
assert source.url == source_url
assert source.backend == 'factory'
assert source.frequency == 'manual'
assert source.active
assert source.owner is None
assert source.organization is None
assert source.validation.state == VALIDATION_PENDING
assert source.validation.on is None
assert source.validation.by is None
assert source.validation.comment is None
def test_error_on_initialize(self):
def init(self):
raise ValueError('test')
source = HarvestSourceFactory(backend='factory')
with assert_emit(signals.before_harvest_job),\
mock_initialize.connected_to(init):
self.action(source.slug)
source.reload()
assert len(HarvestJob.objects(source=source)) == 1
job = source.get_last_job()
assert job.status == 'failed'
assert len(job.errors) == 1
error = job.errors[0]
assert isinstance(error, HarvestError)
assert job.started is not None
assert job.ended is not None
assert len(job.items) == 0
assert len(HarvestJob.objects) == 1
assert len(Dataset.objects) == 0
def test_send_on_delete(self):
reuse = ReuseFactory()
with assert_emit(Reuse.on_delete):
reuse.deleted = datetime.now()
reuse.save()
def test_send_on_delete(self):
dataset = DatasetFactory()
with assert_emit(Dataset.on_delete):
dataset.deleted = datetime.now()
dataset.save()