def test_results_should_not_fail_on_missing_objects(self):
data = []
for _ in range(3):
random_dataset = VisibleDatasetFactory()
data.append(DatasetSearch.serialize(random_dataset))
to_delete_random_dataset = VisibleDatasetFactory()
data.append(DatasetSearch.serialize(to_delete_random_dataset))
search_class = DatasetSearch.temp_search()
search_query = search_class(params={})
service_result = {
"data": data,
"next_page": None,
"page": 1,
"previous_page": None,
"page_size": 20,
"total_pages": 1,
"total": 3
}
search_results = SearchResult(query=search_query,
result=service_result.pop('data'),
**service_result)
to_delete_random_dataset.delete()
assert len(search_results.get_objects()) == 3
# Missing object should be filtered out
objects = search_results.objects
for o in objects:
assert isinstance(o, Dataset)
def test_index_model_from_datetime(self):
kafka_mock = Mock()
KafkaProducerSingleton.get_instance = lambda: kafka_mock
VisibleDatasetFactory(id='61fd30cb29ea95c7bc0e1211',
last_modified=datetime.datetime(2020, 1, 1))
fake_data = VisibleDatasetFactory(id='61fd30cb29ea95c7bc0e1212',
last_modified=datetime.datetime(
2022, 1, 1))
producer = KafkaProducerSingleton.get_instance()
index_model(DatasetSearch,
start=None,
from_datetime=datetime.datetime(2023, 1, 1))
producer.send.assert_not_called()
index_model(DatasetSearch,
start=None,
from_datetime=datetime.datetime(2021, 1, 1))
expected_value = {
'service': 'udata',
'data': DatasetSearch.serialize(fake_data),
'meta': {
'message_type': 'dataset.index',
'index': 'dataset'
}
}
topic = self.app.config['UDATA_INSTANCE_NAME'] + '.dataset.index'
producer.send.assert_called_with(topic,
value=expected_value,
key=b'61fd30cb29ea95c7bc0e1212')
def test_purge_organizations(self):
with self.autoindex():
org = Organization.objects.create(name='delete me',
description='XXX')
resources = [ResourceFactory() for _ in range(2)]
dataset = DatasetFactory(resources=resources, organization=org)
# Upload organization's logo
file = create_test_image()
user = AdminFactory()
self.login(user)
response = self.post(url_for('api.organization_logo', org=org),
{'file': (file, 'test.png')},
json=False)
self.assert200(response)
# Delete organization
response = self.delete(url_for('api.organization', org=org))
self.assert204(response)
tasks.purge_organizations()
# Check organization's logo is deleted
self.assertEqual(list(storages.avatars.list_files()), [])
dataset = Dataset.objects(id=dataset.id).first()
self.assertIsNone(dataset.organization)
organization = Organization.objects(name='delete me').first()
self.assertIsNone(organization)
indexed_dataset = DatasetSearch.get(id=dataset.id,
using=es.client,
index=es.index_name)
self.assertIsNone(indexed_dataset.organization)
def test_results_get_objects(self):
data = []
for _ in range(3):
random_dataset = VisibleDatasetFactory()
data.append(DatasetSearch.serialize(random_dataset))
search_class = DatasetSearch.temp_search()
search_query = search_class(params={})
service_result = {
"data": data,
"next_page": None,
"page": 1,
"previous_page": None,
"page_size": 20,
"total_pages": 1,
"total": 3
}
search_results = SearchResult(query=search_query,
result=service_result.pop('data'),
**service_result)
assert len(search_results.get_objects()) == 3
def test_purge_organizations(self):
with self.autoindex():
org = Organization.objects.create(
name='delete me', deleted='2016-01-01', description='XXX')
resources = [ResourceFactory() for _ in range(2)]
dataset = DatasetFactory(resources=resources, organization=org)
tasks.purge_organizations()
dataset = Dataset.objects(id=dataset.id).first()
self.assertIsNone(dataset.organization)
organization = Organization.objects(name='delete me').first()
self.assertIsNone(organization)
indexed_dataset = DatasetSearch.get(id=dataset.id,
using=es.client,
index=es.index_name)
self.assertIsNone(indexed_dataset.organization)
def test_purge_organizations(self):
with self.autoindex():
org = Organization.objects.create(name='delete me',
deleted='2016-01-01',
description='XXX')
resources = [ResourceFactory() for _ in range(2)]
dataset = DatasetFactory(resources=resources, organization=org)
tasks.purge_organizations()
dataset = Dataset.objects(id=dataset.id).first()
self.assertEqual(dataset.organization, None)
organization = Organization.objects(name='delete me').first()
self.assertEqual(organization, None)
indexed_dataset = DatasetSearch.get(id=dataset.id,
using=es.client,
index=es.index_name)
self.assertEqual(indexed_dataset.organization, '')
def test_producer_should_send_a_message_with_payload_if_indexable(self):
kafka_mock = Mock()
KafkaProducerSingleton.get_instance = lambda: kafka_mock
fake_data = VisibleDatasetFactory(id='61fd30cb29ea95c7bc0e1211')
reindex.run(*as_task_param(fake_data))
producer = KafkaProducerSingleton.get_instance()
expected_value = {
'service': 'udata',
'data': DatasetSearch.serialize(fake_data),
'meta': {
'message_type': 'dataset.index',
'index': 'dataset'
}
}
topic = self.app.config['UDATA_INSTANCE_NAME'] + '.dataset.index'
producer.send.assert_called_with(topic,
value=expected_value,
key=b'61fd30cb29ea95c7bc0e1211')
def test_reindex_model(self):
kafka_mock = Mock()
KafkaProducerSingleton.get_instance = lambda: kafka_mock
fake_data = VisibleDatasetFactory(id='61fd30cb29ea95c7bc0e1211')
producer = KafkaProducerSingleton.get_instance()
index_model(DatasetSearch,
start=datetime.datetime(2022, 2, 20, 20, 2),
reindex=True)
expected_value = {
'service': 'udata',
'data': DatasetSearch.serialize(fake_data),
'meta': {
'message_type': 'dataset.reindex',
'index': 'dataset-2022-02-20-20-02'
}
}
topic = self.app.config['UDATA_INSTANCE_NAME'] + '.dataset.reindex'
producer.send.assert_called_with(topic,
value=expected_value,
key=b'61fd30cb29ea95c7bc0e1211')
from udata.core.dataset.api_fields import dataset_page_fields
from udata.core.dataset.models import Dataset
from udata.core.dataset.search import DatasetSearch
from udata.core.discussions.api import discussion_fields
from udata.core.discussions.models import Discussion
from udata.core.issues.api import issue_fields
from udata.core.issues.models import Issue
from udata.core.reuse.api_fields import reuse_fields
from udata.core.reuse.models import Reuse
from udata.core.storages.api import (uploaded_image_fields, image_parser,
parse_uploaded_image)
ns = api.namespace('organizations', 'Organization related operations')
search_parser = OrganizationSearch.as_request_parser()
dataset_search_parser = DatasetSearch.as_request_parser()
common_doc = {'params': {'org': 'The organization ID or slug'}}
@ns.route('/', endpoint='organizations')
class OrganizationListAPI(API):
'''Organizations collection endpoint'''
@api.doc('list_organizations')
@api.expect(search_parser)
@api.marshal_with(org_page_fields)
def get(self):
'''List or search all organizations'''
search_parser.parse_args()
return search.query(OrganizationSearch, **multi_to_dict(request.args))
def test_purge_organizations(self):
with self.autoindex():
org = Organization.objects.create(name='delete me',
description='XXX')
resources = [ResourceFactory() for _ in range(2)]
dataset = DatasetFactory(resources=resources, organization=org)
# Upload organization's logo
file = create_test_image()
user = AdminFactory()
self.login(user)
response = self.post(url_for('api.organization_logo', org=org),
{'file': (file, 'test.png')},
json=False)
self.assert200(response)
transfer_to_org = Transfer.objects.create(
owner=user,
recipient=org,
subject=dataset,
comment='comment',
)
transfer_from_org = Transfer.objects.create(
owner=org,
recipient=user,
subject=dataset,
comment='comment',
)
oauth_client = OAuth2Client.objects.create(
name='test-client',
owner=user,
organization=org,
redirect_uris=['https://test.org/callback'],
)
# Delete organization
response = self.delete(url_for('api.organization', org=org))
self.assert204(response)
tasks.purge_organizations()
oauth_client.reload()
assert oauth_client.organization is None
assert Transfer.objects.filter(id=transfer_from_org.id).count() == 0
assert Transfer.objects.filter(id=transfer_to_org.id).count() == 0
# Check organization's logo is deleted
self.assertEqual(list(storages.avatars.list_files()), [])
dataset = Dataset.objects(id=dataset.id).first()
self.assertIsNone(dataset.organization)
organization = Organization.objects(name='delete me').first()
self.assertIsNone(organization)
indexed_dataset = DatasetSearch.get(id=dataset.id,
using=es.client,
index=es.index_name)
self.assertIsNone(indexed_dataset.organization)
from udata.core.dataset.api_fields import dataset_page_fields
from udata.core.dataset.models import Dataset
from udata.core.dataset.search import DatasetSearch
from udata.core.discussions.api import discussion_fields
from udata.core.discussions.models import Discussion
from udata.core.issues.api import issue_fields
from udata.core.issues.models import Issue
from udata.core.reuse.api_fields import reuse_fields
from udata.core.reuse.models import Reuse
from udata.core.storages.api import (
uploaded_image_fields, image_parser, parse_uploaded_image
)
ns = api.namespace('organizations', 'Organization related operations')
search_parser = OrganizationSearch.as_request_parser()
dataset_search_parser = DatasetSearch.as_request_parser()
common_doc = {
'params': {'org': 'The organization ID or slug'}
}
@ns.route('/', endpoint='organizations')
class OrganizationListAPI(API):
'''Organizations collection endpoint'''
@api.doc('list_organizations')
@api.expect(search_parser)
@api.marshal_with(org_page_fields)
def get(self):
'''List or search all organizations'''
search_parser.parse_args()