def test_reuse_metrics(self): dataset = VisibleDatasetFactory() reuse = VisibleReuseFactory() issue = IssueFactory(subject=reuse) DiscussionFactory(subject=reuse) reuse.count_datasets() reuse.count_issues() reuse.count_discussions() assert reuse.get_metrics()['datasets'] == 1 assert reuse.get_metrics()['issues'] == 1 assert reuse.get_metrics()['discussions'] == 1 with assert_emit(Reuse.on_update): reuse.datasets.append(dataset) reuse.save() reuse.count_datasets() assert reuse.get_metrics()['datasets'] == 2 dataset.count_reuses() assert dataset.get_metrics()['reuses'] == 1 with assert_emit(Reuse.on_update): reuse.datasets.remove(dataset) reuse.save() dataset_tasks.update_datasets_reuses_metrics() dataset.reload() assert dataset.get_metrics()['reuses'] == 0
def test_add_resource_without_checksum(self): user = UserFactory() dataset = DatasetFactory(owner=user) resource = ResourceFactory(checksum=None) expected_signals = post_save, Dataset.after_save, Dataset.on_update with assert_emit(*expected_signals): dataset.add_resource(ResourceFactory(checksum=None)) assert len(dataset.resources) == 1 with assert_emit(*expected_signals): dataset.add_resource(resource) assert len(dataset.resources) == 2 assert dataset.resources[0].id == resource.id
def test_add_resource_without_checksum(self): user = UserFactory() dataset = DatasetFactory(owner=user) resource = ResourceFactory(checksum=None) expected_signals = post_save, Dataset.after_save, Dataset.on_update with assert_emit(*expected_signals): dataset.add_resource(ResourceFactory(checksum=None)) self.assertEqual(len(dataset.resources), 1) with assert_emit(*expected_signals): dataset.add_resource(resource) self.assertEqual(len(dataset.resources), 2) self.assertEqual(dataset.resources[0].id, resource.id)
def test_add_resource(self): user = UserFactory() dataset = DatasetFactory(owner=user) resource = ResourceFactory() expected_signals = (Dataset.on_resource_added, ) with assert_emit(*expected_signals): dataset.add_resource(ResourceFactory()) assert len(dataset.resources) == 1 with assert_emit(*expected_signals): dataset.add_resource(resource) assert len(dataset.resources) == 2 assert dataset.resources[0].id == resource.id assert dataset.resources[0].dataset == dataset
def test_default(self): org = OrganizationFactory() source = HarvestSourceFactory(backend='factory', organization=org) with assert_emit(signals.before_harvest_job, signals.after_harvest_job): self.action(source.slug) source.reload() assert len(HarvestJob.objects(source=source)) == 1 job = source.get_last_job() assert job.status == 'done' assert job.errors == [] assert job.started is not None assert job.ended is not None assert len(job.items) == COUNT for item in job.items: assert item.status == 'done' assert item.errors == [] assert item.started is not None assert item.ended is not None assert item.dataset is not None dataset = item.dataset assert Dataset.objects(id=dataset.id).first() is not None assert dataset.organization == org assert 'harvest:remote_id' in dataset.extras assert 'harvest:last_update' in dataset.extras assert 'harvest:source_id' in dataset.extras assert len(HarvestJob.objects) == 1 assert len(Dataset.objects) == COUNT
def test_ignore_post_save_signal(self): dataset = DatasetFactory() unexpected_signals = Dataset.after_save, Dataset.on_update with assert_not_emit(*unexpected_signals), assert_emit(post_save): dataset.title = 'New title' dataset.save(signal_kwargs={'ignores': ['post_save']})
def test_add_resource(self): user = UserFactory() dataset = DatasetFactory(owner=user) resource = ResourceFactory() expected_signals = (post_save, Dataset.after_save, Dataset.on_update, Dataset.on_resource_added) with assert_emit(*expected_signals): dataset.add_resource(ResourceFactory()) assert len(dataset.resources) == 1 with assert_emit(*expected_signals): dataset.add_resource(resource) assert len(dataset.resources) == 2 assert dataset.resources[0].id == resource.id assert dataset.resources[0].dataset == dataset
def test_default(self): org = OrganizationFactory() source = HarvestSourceFactory(backend='factory', organization=org) with assert_emit(signals.before_harvest_job, signals.after_harvest_job): self.action(source.slug) source.reload() self.assertEqual(len(HarvestJob.objects(source=source)), 1) job = source.get_last_job() self.assertEqual(job.status, 'done') self.assertEqual(job.errors, []) self.assertIsNotNone(job.started) self.assertIsNotNone(job.ended) self.assertEqual(len(job.items), COUNT) for item in job.items: self.assertEqual(item.status, 'done') self.assertEqual(item.errors, []) self.assertIsNotNone(item.started) self.assertIsNotNone(item.ended) self.assertIsNotNone(item.dataset) dataset = item.dataset self.assertIsNotNone(Dataset.objects(id=dataset.id).first()) self.assertEqual(dataset.organization, org) self.assertIn('harvest:remote_id', dataset.extras) self.assertIn('harvest:last_update', dataset.extras) self.assertIn('harvest:source_id', dataset.extras) self.assertEqual(len(HarvestJob.objects), 1) self.assertEqual(len(Dataset.objects), COUNT)
def test_publish_message_resource_removed(self): kafka_mock = Mock() KafkaProducerSingleton.get_instance = lambda: kafka_mock resource = ResourceFactory() dataset = DatasetFactory(resources=[resource]) expected_signals = (Dataset.on_resource_removed, ) with assert_emit(*expected_signals): dataset.remove_resource(resource) producer = KafkaProducerSingleton.get_instance() message_type = f'resource.{KafkaMessageType.DELETED.value}' expected_value = { 'service': 'udata', 'data': None, 'meta': { 'message_type': message_type, 'dataset_id': str(dataset.id) } } topic = f"{current_app.config['UDATA_INSTANCE_NAME']}.{message_type}" producer.send.assert_called_with(topic, value=expected_value, key=str(resource.id).encode("utf-8"))
def test_ignore_post_save_signal(self): resource = ResourceFactory() DatasetFactory(resources=[resource]) unexpected_signals = Dataset.after_save, Dataset.on_update with assert_not_emit(*unexpected_signals), assert_emit(post_save): resource.title = 'New title' resource.save(signal_kwargs={'ignores': ['post_save']})
def test_create(self): data = self.factory.as_dict() url = url_for('api.organization_badges', org=self.organization) with self.api_user(), assert_emit(on_badge_added): response = self.post(url, data) self.assert201(response) self.organization.reload() self.assertEqual(len(self.organization.badges), 1)
def test_create(self, api): data = self.factory.as_dict() url = url_for('api.organization_badges', org=self.organization) with assert_emit(on_badge_added): response = api.post(url, data) assert201(response) self.organization.reload() assert len(self.organization.badges) is 1
def test_ignore_post_save_signal(self): resource = ResourceFactory() # assigning to a variable to avoid garbage collection issue _ = DatasetFactory(resources=[resource]) unexpected_signals = Dataset.after_save, Dataset.on_update with assert_not_emit(*unexpected_signals), assert_emit(post_save): resource.title = 'New title' resource.save(signal_kwargs={'ignores': ['post_save']})
def test_delete_source_by_objectid(self): source = HarvestSourceFactory() with assert_emit(signals.harvest_source_deleted): deleted_source = actions.delete_source(source.id) self.assertIsNotNone(deleted_source.deleted) self.assertEqual(deleted_source.id, source.id) deleted_sources = HarvestSource.objects(deleted__exists=True) self.assertEqual(len(deleted_sources), 1)
def test_delete_source_by_objectid(self): source = HarvestSourceFactory() with assert_emit(signals.harvest_source_deleted): deleted_source = actions.delete_source(source.id) assert deleted_source.deleted is not None assert deleted_source.id == source.id deleted_sources = HarvestSource.objects(deleted__exists=True) assert len(deleted_sources) == 1
def test_need_update(self): '''It should update the metric on "need_update" signal''' obj = FakeModel.objects.create() metric = FakeMetric(obj) with assert_emit(FakeMetric.need_update, FakeMetric.updated): metric.trigger_update() obj.reload() assert obj.metrics['fake'] == 42
def test_reschedule(self): source = HarvestSourceFactory() with assert_emit(signals.harvest_source_scheduled): source = actions.schedule(str(source.id), hour=0) with assert_emit(signals.harvest_source_scheduled): source = actions.schedule(str(source.id), minute=0) self.assertEqual(len(PeriodicTask.objects), 1) periodic_task = source.periodic_task self.assertEqual(periodic_task, PeriodicTask.objects.first()) self.assertEqual(periodic_task.args, [str(source.id)]) self.assertEqual(periodic_task.crontab.hour, '*') self.assertEqual(periodic_task.crontab.minute, '0') self.assertEqual(periodic_task.crontab.day_of_week, '*') self.assertEqual(periodic_task.crontab.day_of_month, '*') self.assertEqual(periodic_task.crontab.month_of_year, '*') self.assertTrue(periodic_task.enabled) self.assertEqual(periodic_task.name, 'Harvest {0}'.format(source.name))
def test_reschedule(self): source = HarvestSourceFactory() with assert_emit(signals.harvest_source_scheduled): source = actions.schedule(str(source.id), hour=0) with assert_emit(signals.harvest_source_scheduled): source = actions.schedule(str(source.id), minute=0) assert len(PeriodicTask.objects) == 1 periodic_task = source.periodic_task assert periodic_task == PeriodicTask.objects.first() assert periodic_task.args == [str(source.id)] assert periodic_task.crontab.hour == '*' assert periodic_task.crontab.minute == '0' assert periodic_task.crontab.day_of_week == '*' assert periodic_task.crontab.day_of_month == '*' assert periodic_task.crontab.month_of_year == '*' assert periodic_task.enabled assert periodic_task.name == 'Harvest {0}'.format(source.name)
def test_update(self, app): '''It should store the updated metric on "updated" signal''' with assert_emit(FakeSiteMetric.need_update, FakeSiteMetric.updated): FakeSiteMetric.update() metrics = Metrics.objects.last_for(app.config['SITE_ID']) assert metrics.values['fake'] == FAKE_VALUE site = Site.objects.get(id=app.config['SITE_ID']) assert site.metrics['fake'] == FAKE_VALUE
def test_create_same(self): data = self.factory.as_dict() url = url_for('api.organization_badges', org=self.organization) with self.api_user(): with assert_emit(on_badge_added): self.post(url, data) with assert_not_emit(on_badge_added): response = self.post(url, data) self.assertStatus(response, 200) self.organization.reload() self.assertEqual(len(self.organization.badges), 1)
def test_create_source_with_config(self): source_url = faker.url() config = {'filters': [{'key': 'test', 'value': 42}]} with assert_emit(signals.harvest_source_created): source = actions.create_source('Test source', source_url, 'factory', config=config) assert source.config == config
def test_update_source(self): source = HarvestSourceFactory() data = source.to_dict() new_url = faker.url() data['url'] = new_url with assert_emit(signals.harvest_source_updated): source = actions.update_source(source.id, data) assert source.url == new_url source.reload() assert source.url == new_url
def test_updated(self): '''It should store the updated metric on "updated" signal''' obj = FakeModel.objects.create() metric = FakeMetric(obj) metric.value = 'some-value' with assert_emit(FakeMetric.updated): with assert_not_emit(FakeMetric.need_update): metric.notify_update() metrics = Metrics.objects.last_for(obj) assert metrics.values['fake'] == 'some-value'
def test_update_source(self): source = HarvestSourceFactory() data = source.to_dict() new_url = faker.url() data['url'] = new_url with assert_emit(signals.harvest_source_updated): source = actions.update_source(source.id, data) self.assertEqual(source.url, new_url) source.reload() self.assertEqual(source.url, new_url)
def test_delete(self, api): badge = self.factory() self.organization.add_badge(badge.kind) self.organization.save() url = url_for('api.organization_badge', org=self.organization, badge_kind=str(badge.kind)) with assert_emit(on_badge_removed): response = api.delete(url) assert204(response) self.organization.reload() assert len(self.organization.badges) is 0
def test_update_resource(self): user = UserFactory() resource = ResourceFactory() dataset = DatasetFactory(owner=user, resources=[resource]) expected_signals = post_save, Dataset.after_save, Dataset.on_update resource.description = 'New description' with assert_emit(*expected_signals): dataset.update_resource(resource) self.assertEqual(len(dataset.resources), 1) self.assertEqual(dataset.resources[0].id, resource.id) self.assertEqual(dataset.resources[0].description, 'New description')
def test_update_resource(self): user = UserFactory() resource = ResourceFactory() dataset = DatasetFactory(owner=user, resources=[resource]) expected_signals = post_save, Dataset.after_save, Dataset.on_update resource.description = 'New description' with assert_emit(*expected_signals): dataset.update_resource(resource) assert len(dataset.resources) == 1 assert dataset.resources[0].id == resource.id assert dataset.resources[0].description == 'New description'
def test_update_resource(self): user = UserFactory() resource = ResourceFactory() dataset = DatasetFactory(owner=user, resources=[resource]) expected_signals = (Dataset.on_resource_updated, ) resource.description = 'New description' with assert_emit(*expected_signals): dataset.update_resource(resource) assert len(dataset.resources) == 1 assert dataset.resources[0].id == resource.id assert dataset.resources[0].description == 'New description'
def test_delete(self): badge = self.factory() self.organization.badges.append(badge) self.organization.save() url = url_for('api.organization_badge', org=self.organization, badge_kind=str(badge.kind)) with self.api_user(): with assert_emit(on_badge_removed): response = self.delete(url) self.assertStatus(response, 204) self.organization.reload() self.assertEqual(len(self.organization.badges), 0)
def test_unschedule(self): periodic_task = PeriodicTask.objects.create( task='harvest', name=faker.name(), description=faker.sentence(), enabled=True, crontab=PeriodicTask.Crontab()) source = HarvestSourceFactory(periodic_task=periodic_task) with assert_emit(signals.harvest_source_unscheduled): actions.unschedule(str(source.id)) source.reload() self.assertEqual(len(PeriodicTask.objects), 0) self.assertIsNone(source.periodic_task)
def test_create_source_with_config(self): source_url = faker.url() config = { 'filters': [{'key': 'test', 'value': 42}], 'features': {'key': True}, } with assert_emit(signals.harvest_source_created): source = actions.create_source('Test source', source_url, 'factory', config=config) assert source.config == config
def test_unschedule(self): periodic_task = PeriodicTask.objects.create( task='harvest', name=faker.name(), description=faker.sentence(), enabled=True, crontab=PeriodicTask.Crontab() ) source = HarvestSourceFactory(periodic_task=periodic_task) with assert_emit(signals.harvest_source_unscheduled): actions.unschedule(str(source.id)) source.reload() assert len(PeriodicTask.objects) == 0 assert source.periodic_task is None
def test_empty(self): source = HarvestSourceFactory(backend='factory', config={'count': 0}) with assert_emit(signals.before_harvest_job, signals.after_harvest_job): self.action(source.slug) source.reload() self.assertEqual(len(HarvestJob.objects(source=source)), 1) job = source.get_last_job() self.assertEqual(job.status, 'done') self.assertEqual(job.errors, []) self.assertIsNotNone(job.started) self.assertIsNotNone(job.ended) self.assertEqual(job.items, []) self.assertEqual(len(HarvestJob.objects), 1) self.assertEqual(len(Dataset.objects), 0)
def test_empty(self): source = HarvestSourceFactory(backend='factory', config={'count': 0}) with assert_emit(signals.before_harvest_job, signals.after_harvest_job): self.action(source.slug) source.reload() assert len(HarvestJob.objects(source=source)) == 1 job = source.get_last_job() assert job.status == 'done' assert job.errors == [] assert job.started is not None assert job.ended is not None assert job.items == [] assert len(HarvestJob.objects) == 1 assert len(Dataset.objects) == 0
def test_create_source(self): source_url = faker.url() with assert_emit(signals.harvest_source_created): source = actions.create_source('Test source', source_url, 'factory') self.assertEqual(source.name, 'Test source') self.assertEqual(source.slug, 'test-source') self.assertEqual(source.url, source_url) self.assertEqual(source.backend, 'factory') self.assertEqual(source.frequency, 'manual') self.assertTrue(source.active) self.assertIsNone(source.owner) self.assertIsNone(source.organization) self.assertEqual(source.validation.state, VALIDATION_PENDING) self.assertIsNone(source.validation.on) self.assertIsNone(source.validation.by) self.assertIsNone(source.validation.comment)
def test_error_on_item(self): def process(self, item): if item.remote_id == '1': raise ValueError('test') source = HarvestSourceFactory(backend='factory') with assert_emit(signals.before_harvest_job, signals.after_harvest_job), \ mock_process.connected_to(process): self.action(source.slug) source.reload() assert len(HarvestJob.objects(source=source)) == 1 job = source.get_last_job() assert job.status == 'done-errors' assert job.started is not None assert job.ended is not None assert len(job.errors) == 0 assert len(job.items) == COUNT items_ok = filter(lambda i: not len(i.errors), job.items) assert len(items_ok) == COUNT - 1 for item in items_ok: assert item.started is not None assert item.ended is not None assert item.status == 'done' assert item.errors == [] item_ko = filter(lambda i: len(i.errors), job.items)[0] assert item_ko.started is not None assert item_ko.ended is not None assert item_ko.status == 'failed' assert len(item_ko.errors) == 1 error = item_ko.errors[0] assert isinstance(error, HarvestError) assert len(HarvestJob.objects) == 1 assert len(Dataset.objects) == COUNT - 1
def test_create_source(self): source_url = faker.url() with assert_emit(signals.harvest_source_created): source = actions.create_source('Test source', source_url, 'factory') assert source.name == 'Test source' assert source.slug == 'test-source' assert source.url == source_url assert source.backend == 'factory' assert source.frequency == 'manual' assert source.active assert source.owner is None assert source.organization is None assert source.validation.state == VALIDATION_PENDING assert source.validation.on is None assert source.validation.by is None assert source.validation.comment is None
def test_error_on_initialize(self): def init(self): raise ValueError('test') source = HarvestSourceFactory(backend='factory') with assert_emit(signals.before_harvest_job),\ mock_initialize.connected_to(init): self.action(source.slug) source.reload() assert len(HarvestJob.objects(source=source)) == 1 job = source.get_last_job() assert job.status == 'failed' assert len(job.errors) == 1 error = job.errors[0] assert isinstance(error, HarvestError) assert job.started is not None assert job.ended is not None assert len(job.items) == 0 assert len(HarvestJob.objects) == 1 assert len(Dataset.objects) == 0
def test_send_on_delete(self): reuse = ReuseFactory() with assert_emit(Reuse.on_delete): reuse.deleted = datetime.now() reuse.save()
def test_send_on_delete(self): dataset = DatasetFactory() with assert_emit(Dataset.on_delete): dataset.deleted = datetime.now() dataset.save()