def test_by_id(self, client):
nd = factories.NormalizedDataFactory(data={'@graph': []})
resp = client.get('/api/v2/normalizeddata/{}/'.format(IDObfuscator.encode(nd)))
assert resp.status_code == 200
assert resp.json()['data']['id'] == IDObfuscator.encode(nd)
assert resp.json()['data']['type'] == 'NormalizedData'
assert resp.json()['data']['attributes']['data'] == {'@graph': []}
def test_some_unprocessed_date(self):
rds = factories.RawDatumFactory.create_batch(55)
for rd in rds[:25]:
factories.NormalizedDataFactory(raw=rd)
assert rawdata_janitor() == 30
assert IngestJob.objects.count() == 30
def test_some_unprocessed_date(self, mock_transform):
rds = factories.RawDatumFactory.create_batch(55)
for rd in rds[:25]:
factories.NormalizedDataFactory(raw=rd)
assert rawdata_janitor() == 30
assert sorted(mock_transform.call_args_list) == sorted(
[mock.call((rd.id, ), throw=True, retries=4) for rd in rds[25:]])
def test_idempotent(self):
rds = factories.RawDatumFactory.create_batch(55)
for rd in rds:
factories.NormalizedDataFactory(raw=rd)
assert rawdata_janitor() == 0
assert rawdata_janitor() == 0
assert IngestJob.objects.count() == 0
def test_does_not_set_no_output(self):
raw = factories.RawDatumFactory(datum=json.dumps({'@graph': []}))
factories.NormalizedDataFactory(raw=raw)
tasks.transform(raw.id)
raw.refresh_from_db()
assert raw.no_output is None
def test_idempotent(self, mock_transform):
rds = factories.RawDatumFactory.create_batch(55)
for rd in rds:
factories.NormalizedDataFactory(raw=rd)
assert rawdata_janitor() == 0
assert rawdata_janitor() == 0
assert mock_transform.call_args_list == []
def _ingest(graph, disambiguate=True, regulate=True, user=None, save=True):
if regulate:
Regulator().regulate(graph)
nd = factories.NormalizedDataFactory(
source=user) if user else normalized_data
cs = ChangeSetBuilder(graph, nd,
disambiguate=disambiguate).build_change_set()
if save and cs is not None:
cs.accept()
return cs
def test_elastic_queue_only_works(self, celery_app):
nd = factories.NormalizedDataFactory(data={
'@graph': [{
'@id': '_:1234',
'@type': 'person',
'given_name': 'Jane',
'family_name': 'Doe',
}]
},
source__is_trusted=True)
with celery_app.pool.acquire(block=True) as connection:
with connection.SimpleQueue(
settings.ELASTIC_QUEUE,
**settings.ELASTIC_QUEUE_SETTINGS) as queue:
queue.clear()
celery_app.tasks['share.tasks.disambiguate'](nd.id)
with pytest.raises(Empty):
queue.get(timeout=5)
def test_elastic_queue(self, celery_app):
nd = factories.NormalizedDataFactory(data={
'@graph': [{
'@id': '_:1234',
'@type': 'creativework',
'title': 'All About Tamanduas',
}]
},
source__is_trusted=True)
with celery_app.pool.acquire(block=True) as connection:
with connection.SimpleQueue(
settings.ELASTIC_QUEUE,
**settings.ELASTIC_QUEUE_SETTINGS) as queue:
queue.clear()
celery_app.tasks['share.tasks.disambiguate'](nd.id)
message = queue.get(timeout=5)
assert len(
apps.get_model('share',
message.payload['model']).objects.filter(
id__in=message.payload['ids'])) == len(
message.payload['ids'])