def test_update_dependencies_accept(self, john_doe, normalized_data_id):
graph = ChangeGraph([{
'@id': IDObfuscator.encode(john_doe),
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {
'@id': IDObfuscator.encode(john_doe),
'@type': 'person'
},
'creative_work': {
'@id': '_:789',
'@type': 'preprint'
},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = models.ChangeSet.objects.from_graph(
graph, normalized_data_id)
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(
agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(
agent_relations__agent=john_doe).first().title == 'All About Cats'
def test_no_icon(self, schema):
x = factories.AbstractCreativeWorkFactory()
source = factories.SourceFactory(icon='')
x.sources.add(source.user)
# Have to use % formats because of {}s everywhere
result = schema.execute('''
query {
creativeWork(id: "%s") {
id,
title,
description,
sources {
title
}
}
}
''' % (IDObfuscator.encode(x), ))
assert not result.errors
assert result.data == OrderedDict([
('creativeWork', OrderedDict([
('id', IDObfuscator.encode(x)),
('title', x.title),
('description', x.description),
('sources', [])
]))
])
def test_by_id(self, client):
nd = factories.NormalizedDataFactory(data={'@graph': []})
resp = client.get('/api/v2/normalizeddata/{}/'.format(IDObfuscator.encode(nd)))
assert resp.status_code == 200
assert resp.json()['data']['id'] == IDObfuscator.encode(nd)
assert resp.json()['data']['type'] == 'NormalizedData'
assert resp.json()['data']['attributes']['data'] == {'@graph': []}
def test_update_dependencies_accept(self, john_doe, normalized_data_id):
graph = ChangeGraph([{
'@id': IDObfuscator.encode(john_doe),
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {'@id': IDObfuscator.encode(john_doe), '@type': 'person'},
'creative_work': {'@id': '_:789', '@type': 'preprint'},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = models.ChangeSet.objects.from_graph(graph, normalized_data_id)
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).first().title == 'All About Cats'
def test_no_icon(self):
x = factories.AbstractCreativeWorkFactory()
source = factories.SourceFactory(icon='')
x.sources.add(source.user)
# Have to use % formats because of {}s everywhere
result = schema.execute('''
query {
creativeWork(id: "%s") {
id,
title,
description,
sources {
title
}
}
}
''' % (IDObfuscator.encode(x), ))
assert not result.errors
assert result.data == OrderedDict([
('creativeWork', OrderedDict([
('id', IDObfuscator.encode(x)),
('title', x.title),
('description', x.description),
('sources', [])
]))
])
def test_change_diff(self, graph):
tag = factories.TagFactory(name='tag1')
assert graph.create(IDObfuscator.encode(tag), 'tag', {
'name': 'tag2'
}).change == {
'name': 'tag2'
}
def test_list_with_items(self, client):
user = ShareUserFactory()
banner = SiteBanner.objects.create(
title='Why wasnt I there',
description='I could have saved them',
created_by=user,
last_modified_by=user,
)
resp = client.get('/api/v2/site_banners/')
assert resp.status_code == 200
assert resp.json() == {
'data': [{
'id': IDObfuscator.encode(banner),
'type': 'SiteBanner',
'attributes': {
'color': 'info',
'icon': 'exclamation',
'title': 'Why wasnt I there',
'description': 'I could have saved them',
}
}],
'links': {
'first': 'http://testserver/api/v2/site_banners/?page=1',
'last': 'http://testserver/api/v2/site_banners/?page=1',
'next': None,
'prev': None,
},
'meta': {
'pagination': {'count': 1, 'pages': 1, 'page': 1},
}
}
def test_can_delete_work(self, client, normalized_data_id):
preprint = factories.AbstractCreativeWorkFactory(is_deleted=False)
preprint.administrative_change(type='share.dataset')
assert preprint.is_deleted is False
encoded_id = IDObfuscator.encode(preprint)
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 200
preprint.administrative_change(is_deleted=True)
assert preprint.is_deleted is True
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 403
assert response.json() == {
"errors": [{
"source": {
"pointer": "/data"
},
"detail": "This data set has been removed.",
"status": "403"
}]
}
response = client.get('/api/v2/datasets/')
assert response.status_code == 200
assert response.json() == {
'data': [],
'links': {
'next': None,
'prev': None
}
}
def test_update_dependencies_accept(self, john_doe, normalized_data):
john_doe_id = IDObfuscator.encode(john_doe)
graph = MutableGraph.from_jsonld([{
'@id': john_doe_id,
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {'@id': john_doe_id, '@type': 'person'},
'creative_work': {'@id': '_:789', '@type': 'preprint'},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = ChangeSetBuilder(graph, normalized_data, matches={
john_doe_id: john_doe,
}).build_change_set()
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).first().title == 'All About Cats'
def test_get_record(self, post, all_about_anteaters):
ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
parsed = oai_request({'verb': 'GetRecord', 'metadataPrefix': 'oai_dc', 'identifier': ant_id}, post)
records = parsed.xpath('//ns0:GetRecord/ns0:record', namespaces=NAMESPACES)
assert len(records) == 1
assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text
assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data,
context={
'request': request
})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(
request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job).order_by('-created_at').values_list(
'id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response(
{
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task':
async_result.id,
'ingest_job':
request.build_absolute_uri(
reverse('api:ingestjob-detail',
args=[IDObfuscator.encode(ingester.job)])),
}
},
status=status.HTTP_202_ACCEPTED)
def test_list_with_items(self, client):
user = ShareUserFactory()
banner = SiteBanner.objects.create(
title='Why wasnt I there',
description='I could have saved them',
created_by=user,
last_modified_by=user,
)
resp = client.get('/api/v2/site_banners/')
assert resp.status_code == 200
assert resp.json() == {
'data': [{
'id': IDObfuscator.encode(banner),
'type': 'SiteBanner',
'attributes': {
'color': 'info',
'icon': 'exclamation',
'title': 'Why wasnt I there',
'description': 'I could have saved them',
}
}],
'links': {
'first': 'http://testserver/api/v2/site_banners/?page=1',
'last': 'http://testserver/api/v2/site_banners/?page=1',
'next': None,
'prev': None,
},
'meta': {
'pagination': {
'count': 1,
'pages': 1,
'page': 1
},
}
}
def test_add_multiple_sources(self, celery_app):
source1 = factories.SourceFactory()
source2 = factories.SourceFactory()
work = factories.AbstractCreativeWorkFactory(title='All about Canada')
data = {
'@id': IDObfuscator.encode(work),
'@type': 'creativework',
'title': 'All aboot Canada'
}
nd1 = NormalizedData.objects.create(source=source1.user,
data={'@graph': [data]})
nd2 = NormalizedData.objects.create(source=source2.user,
data={'@graph': [data]})
assert work.sources.count() == 0
celery_app.tasks['share.tasks.disambiguate'](nd1.id)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 1
celery_app.tasks['share.tasks.disambiguate'](nd2.id)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 2
def get_url(self, obj, view_name, request, format):
obfuscated_id = IDObfuscator.encode(obj)
kwargs = {self.lookup_url_kwarg: obfuscated_id}
return self.reverse(view_name,
kwargs=kwargs,
request=request,
format=format)
def test_list_records(self, post, all_about_anteaters, django_assert_num_queries):
with django_assert_num_queries(2):
parsed = oai_request({'verb': 'ListRecords', 'metadataPrefix': 'oai_dc'}, post)
records = parsed.xpath('//ns0:ListRecords/ns0:record', namespaces=NAMESPACES)
assert len(records) == 1
assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text
ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text
def refs(n):
if isinstance(n, list):
return [refs(node) for node in n]
instance = self._get_match(n)
return {
'@id': IDObfuscator.encode(instance) if instance else n.id,
'@type': n.type,
}
def refs(n):
if isinstance(n, list):
return [refs(node) for node in n]
instance = self._get_match(n)
return {
'@id': IDObfuscator.encode(instance) if instance else n.id,
'@type': n.type,
}
def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(
IDObfuscator.encode(tag), 'tag', {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}).change == {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}
def assert_indexed(self):
encoded_id = IDObfuscator.encode(self.work)
doc = self.elastic.es_client.get(index=self.elastic.es_index,
doc_type='creativeworks',
id=encoded_id)
assert doc['found'] is True
assert doc['_id'] == encoded_id
return doc
def delete_works(self, queryset, source_config, dry_run):
for work in queryset:
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(
source_config.source.name, IDObfuscator.encode(work))),
work.title))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(
IDObfuscator.encode(work))), work.title))
if not dry_run:
work.administrative_change(is_deleted=True)
def test_ignore_generic_work_type(self, change_factory, all_about_anteaters):
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(all_about_anteaters),
'@type': 'creativework'
}]
}, disambiguate=True)
assert cs is None
def oai_identifier(self, work):
if isinstance(work, int):
share_id = IDObfuscator.encode_id(work, AbstractCreativeWork)
else:
share_id = IDObfuscator.encode(work)
return 'oai{delim}{repository}{delim}{id}'.format(
id=share_id,
repository=self.REPOSITORY_IDENTIFIER,
delim=self.IDENTIFER_DELIMITER)
def assert_indexed(self):
encoded_id = IDObfuscator.encode(self.work)
doc = self.elastic.es_client.get(
index=self.elastic.es_index,
doc_type='creativeworks',
id=encoded_id
)
assert doc['found'] is True
assert doc['_id'] == encoded_id
return doc
def test_oai_errors(self, verb, params, errors, post, all_about_anteaters):
ant_id = IDObfuscator.encode(all_about_anteaters)
data = {
'verb': verb,
**{k: v.format(id=ant_id)
for k, v in params.items()}
}
actual_errors = oai_request(data, post, errors=True)
for error_code in errors:
assert any(
e.attrib.get('code') == error_code for e in actual_errors)
def delete_works(self, works_qs, source_config, dry_run, superfluous,
delete_related):
works_deleted = []
if not superfluous:
works_qs = works_qs.filter(is_deleted=False)
for work in works_qs.prefetch_related('sources'):
works_deleted.append(work.id)
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(
source_config.source.name, IDObfuscator.encode(work))),
work.title))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(
IDObfuscator.encode(work))), work.title))
if not dry_run:
work.administrative_change(is_deleted=True)
self.stdout.write('\nProcessed {} works!'.format(len(works_deleted)),
style_func=self.style.SUCCESS)
if not delete_related:
return
self.stdout.write('\nNow deleting related works...\n')
related_works = AbstractCreativeWork.objects.filter(
Q(incoming_creative_work_relations__subject_id__in=works_deleted) |
Q(outgoing_creative_work_relations__related_id__in=works_deleted),
is_deleted=False,
sources__id=source_config.source.user_id).prefetch_related(
'sources')
# Traverse related works only one level deep, please
self.delete_works(related_works, source_config, dry_run, superfluous,
False)
def test_list_identifiers(self, post, all_about_anteaters):
parsed = oai_request(
{
'verb': 'ListIdentifiers',
'metadataPrefix': 'oai_dc'
}, post)
identifiers = parsed.xpath(
'//ns0:ListIdentifiers/ns0:header/ns0:identifier',
namespaces=NAMESPACES)
assert len(identifiers) == 1
assert identifiers[0].text == 'oai:share.osf.io:{}'.format(
IDObfuscator.encode(all_about_anteaters))
def test_by_id(self, client):
source = Source.objects.first()
resp = client.get('{}{}/'.format(self.endpoint, IDObfuscator.encode(source)))
assert resp.status_code == 200
assert IDObfuscator.load(resp.json()['data']['id']) == source
assert resp.json()['data']['type'] == 'Source'
assert resp.json()['data']['attributes'] == {
'name': source.name,
'icon': 'http://testserver{}'.format(source.icon.url),
'homePage': source.home_page,
'longTitle': source.long_title,
}
def test_by_id(self, client):
source = Source.objects.first()
resp = client.get('{}{}/'.format(self.endpoint, IDObfuscator.encode(source)))
assert resp.status_code == 200
assert IDObfuscator.load(resp.json()['data']['id']) == source
assert resp.json()['data']['type'] == 'Source'
assert resp.json()['data']['attributes'] == {
'name': source.name,
'icon': 'http://testserver{}'.format(source.icon.url),
'homePage': source.home_page,
'longTitle': source.long_title,
}
def test_get_by_id(self, trusted_user, client):
reg = models.ProviderRegistration.objects.create(
contact_name='Tester Testington',
contact_email='*****@*****.**',
contact_affiliation='Just some person',
source_name='PlsWerk',
submitted_by=trusted_user,
)
resp = client.get(
'/api/v2/sourceregistrations/{}/'.format(IDObfuscator.encode(reg)),
content_type='application/vnd.api+json',
HTTP_AUTHORIZATION='Bearer {}'.format(
trusted_user.accesstoken_set.first()),
)
assert resp.status_code == 200
assert resp.json()['data']['id'] == IDObfuscator.encode(reg)
assert resp.json()['data']['type'] == 'ProviderRegistration'
assert resp.json()['data']['attributes'] == {
'contactAffiliation': 'Just some person',
'contactEmail': '*****@*****.**',
'contactName': 'Tester Testington',
'directSource': False,
'sourceAdditionalInfo': '',
'sourceBaseUrl': '',
'sourceDescription': '',
'sourceDisallowedSets': '',
'sourceDocumentation': '',
'sourceName': 'PlsWerk',
'sourceOai': False,
'sourcePreferredMetadataPrefix': '',
'sourceRateLimit': '',
'status': 'pending',
'submittedAt': resp.json()['data']['attributes']['submittedAt']
}
def test_change_existing(self, change_factory, jane_doe):
change_set = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(jane_doe),
'@type': 'Person',
'given_name': 'John'
}]
}, disambiguate=True)
assert change_set.changes.first().target == jane_doe
assert jane_doe.given_name == 'Jane'
change_set.accept()
jane_doe.refresh_from_db()
assert jane_doe.given_name == 'John'
def test_date_updated_update(self, change_ids, change_factory, all_about_anteaters):
"""
Submitting an identical date as a string should be recognized as no change.
"""
all_about_anteaters.date_updated = pendulum.now()
all_about_anteaters.change_id = change_ids.get()
all_about_anteaters.save()
change_set = change_factory.from_graph({
'@graph': [{
'@type': 'article',
'@id': IDObfuscator.encode(all_about_anteaters),
'date_updated': str(all_about_anteaters.date_updated)
}]
}, disambiguate=True)
assert change_set is None
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data,
context={
'request': request
})
if serializer.is_valid(raise_exception=True):
nm_instance = serializer.save()
async_result = disambiguate.delay(nm_instance.id)
# TODO Fix Me
return Response(
{
'id': IDObfuscator.encode(nm_instance),
'type': 'NormalizedData',
'attributes': {
'task': async_result.id
}
},
status=status.HTTP_202_ACCEPTED)
def test_work_type_stays_nongeneric(self, change_factory, all_about_anteaters):
new_title = 'Some about Anteaters'
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(all_about_anteaters),
'@type': 'creativework',
'title': new_title
}]
}, disambiguate=True)
assert all_about_anteaters.type == 'share.article'
assert models.Publication.objects.count() == 1
cs.accept()
all_about_anteaters.refresh_from_db()
assert all_about_anteaters.type == 'share.article'
assert all_about_anteaters.title == new_title
def test_change_agent_type(self, change_factory, university_of_whales):
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(university_of_whales),
'@type': 'consortium'
}]
}, disambiguate=True)
assert models.Institution.objects.count() == 1
assert models.Consortium.objects.count() == 0
(org,) = cs.accept()
assert org.type == 'share.consortium'
assert org.id == university_of_whales.id
assert org.name == university_of_whales.name
assert models.Institution.objects.count() == 0
assert models.Consortium.objects.count() == 1
def test_can_delete_work(self, client, normalized_data_id):
preprint = factories.AbstractCreativeWorkFactory(is_deleted=False)
preprint.administrative_change(type='share.dataset')
assert preprint.is_deleted is False
encoded_id = IDObfuscator.encode(preprint)
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 200
preprint.administrative_change(is_deleted=True)
assert preprint.is_deleted is True
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 403
assert response.json() == {"errors": [{"source": {"pointer": "/data"}, "detail": "This data set has been removed.", "status": "403"}]}
response = client.get('/api/v2/datasets/')
assert response.status_code == 200
assert response.json() == {'data': [], 'links': {'next': None, 'prev': None}}
def test_add_multiple_sources(self):
source1 = factories.SourceFactory()
source2 = factories.SourceFactory()
work = factories.AbstractCreativeWorkFactory(title='All about Canada')
data = [{'@id': IDObfuscator.encode(work), '@type': 'creativework', 'title': 'All aboot Canada'}]
assert work.sources.count() == 0
Ingester(data).as_user(source1.user).ingest(index=False)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 1
Ingester(data).as_user(source2.user).ingest(index=False)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 2
def test_order(self, client, order):
resp = client.get('/api/v2/atom', {'order': order})
assert resp.status_code == 200
feed = etree.fromstring(resp.content)
works = AbstractCreativeWork.objects.order_by('-' + order).exclude(
**{order: None})
assert len(feed.xpath('//atom:entry', namespaces=NAMESPACES)) == 11
for creative_work, entry in zip(
works,
feed.xpath('//atom:entry',
namespaces={'atom':
'http://www.w3.org/2005/Atom'})):
assert entry.find(
'atom:title',
namespaces=NAMESPACES).text == creative_work.title
assert entry.find(
'atom:summary',
namespaces=NAMESPACES).text == creative_work.description
assert entry.find('atom:link',
namespaces=NAMESPACES).attrib['href'].endswith(
IDObfuscator.encode(creative_work))
if getattr(creative_work, order):
assert entry.find(
'atom:updated', namespaces=NAMESPACES).text == getattr(
creative_work,
order).replace(microsecond=0).isoformat()
else:
assert entry.find('atom:updated',
namespaces=NAMESPACES).text is None
if creative_work.date_published:
assert entry.find(
'atom:published', namespaces=NAMESPACES
).text == creative_work.date_published.isoformat()
else:
assert entry.find('atom:published',
namespaces=NAMESPACES) is None
def test_get_data(self, generator, model, route, controlled_values, client, Graph, ingest):
ingest(Graph(initial))
graph = Graph(*generator)
Regulator().regulate(graph)
matches = Matcher(DatabaseStrategy()).find_all_matches(graph)
for node in graph:
if node.type == model:
expected = node
expected_id = IDObfuscator.encode(matches[node])
break
response = client.get('/api/v2/{}/{}/'.format(route, expected_id))
actual = json.loads(response.content.decode(encoding='UTF-8'))
assert response.status_code == 200
assert actual['data']['id'] == expected_id
assert actual['data']['attributes']['type'] == expected.type
for value in controlled_values:
assert actual['data']['attributes'][value] == expected[camelCase_to_underscore(value)]
def post(self, request, *args, **kwargs):
try:
jsonschema.validate(request.data, v1_schemas.v1_push_schema)
except (jsonschema.exceptions.ValidationError) as error:
raise ParseError(detail=error.message)
try:
prelim_data = request.data['jsonData']
except ParseError as error:
return Response('Invalid JSON - {0}'.format(error.message),
status=status.HTTP_400_BAD_REQUEST)
# store raw data, assuming you can only submit one at a time
with transaction.atomic():
try:
doc_id = prelim_data['uris']['canonicalUri']
except KeyError:
return Response(
{
'errors': 'Canonical URI not found in uris.',
'data': prelim_data
},
status=status.HTTP_400_BAD_REQUEST)
ingester = Ingester(prelim_data,
doc_id).as_user(request.user,
'v1_push').ingest_async()
return Response(
{
'task_id':
ingester.async_task.id,
'ingest_job':
request.build_absolute_uri(
reverse('api:ingestjob-detail',
args=[IDObfuscator.encode(ingester.job)])),
},
status=status.HTTP_202_ACCEPTED)
def delete_works(self, works_qs, source_config, dry_run, superfluous, delete_related):
works_deleted = []
if not superfluous:
works_qs = works_qs.filter(is_deleted=False)
for work in works_qs.prefetch_related('sources'):
works_deleted.append(work.id)
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(source_config.source.name, IDObfuscator.encode(work))),
work.title
))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(IDObfuscator.encode(work))),
work.title
))
if not dry_run:
work.administrative_change(is_deleted=True)
self.stdout.write('\nProcessed {} works!'.format(len(works_deleted)), style_func=self.style.SUCCESS)
if not delete_related:
return
self.stdout.write('\nNow deleting related works...\n')
related_works = AbstractCreativeWork.objects.filter(
Q(incoming_creative_work_relations__subject_id__in=works_deleted) | Q(outgoing_creative_work_relations__related_id__in=works_deleted),
is_deleted=False,
sources__id=source_config.source.user_id
).prefetch_related('sources')
# Traverse related works only one level deep, please
self.delete_works(related_works, source_config, dry_run, superfluous, False)
def test_order(self, client, order):
resp = client.get('/api/v2/atom', {'order': order})
assert resp.status_code == 200
feed = etree.fromstring(resp.content)
works = AbstractCreativeWork.objects.order_by('-' + order).exclude(**{order: None})
assert len(feed.xpath('//atom:entry', namespaces=NAMESPACES)) == 11
for creative_work, entry in zip(works, feed.xpath('//atom:entry', namespaces={'atom': 'http://www.w3.org/2005/Atom'})):
try:
contributors = list(AbstractAgentWorkRelation.objects.filter(creative_work_id=creative_work.id))
first_contributor = AbstractAgentWorkRelation.objects.get(creative_work_id=creative_work.id, order_cited=0)
except Exception:
contributors = None
assert entry.find('atom:title', namespaces=NAMESPACES).text == creative_work.title
assert entry.find('atom:summary', namespaces=NAMESPACES).text == creative_work.description
assert entry.find('atom:link', namespaces=NAMESPACES).attrib['href'].endswith(IDObfuscator.encode(creative_work))
if not contributors:
assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == 'No authors provided.'
elif len(contributors) > 1:
assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == '{} et al.'.format(first_contributor.agent.name)
else:
assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == first_contributor.agent.name
if getattr(creative_work, order):
assert entry.find('atom:updated', namespaces=NAMESPACES).text == getattr(creative_work, order).replace(microsecond=0).isoformat()
else:
assert entry.find('atom:updated', namespaces=NAMESPACES).text is None
if creative_work.date_published:
assert entry.find('atom:published', namespaces=NAMESPACES).text == creative_work.date_published.isoformat()
else:
assert entry.find('atom:published', namespaces=NAMESPACES) is None
def encoded_id(self, obj):
return IDObfuscator.encode(obj)
def serialize_subject(self, subject):
return {
'id': IDObfuscator.encode(subject),
'type': 'subject',
'name': safe_substr(subject.name),
}
def post(self, request, *args, **kwargs):
try:
jsonschema.validate(request.data, v1_schemas.v1_push_schema)
except (jsonschema.exceptions.ValidationError) as error:
raise ParseError(detail=error.message)
try:
prelim_data = request.data['jsonData']
except ParseError as error:
return Response(
'Invalid JSON - {0}'.format(error.message),
status=status.HTTP_400_BAD_REQUEST
)
# store raw data, assuming you can only submit one at a time
with transaction.atomic():
try:
doc_id = prelim_data['uris']['canonicalUri']
except KeyError:
return Response({'errors': 'Canonical URI not found in uris.', 'data': prelim_data}, status=status.HTTP_400_BAD_REQUEST)
ingester = Ingester(prelim_data, doc_id).as_user(request.user, 'v1_push').ingest_async(urgent=True)
return Response({
'task_id': ingester.async_task.id,
'ingest_job': request.build_absolute_uri(reverse('api:ingestjob-detail', args=[IDObfuscator.encode(ingester.job)])),
}, status=status.HTTP_202_ACCEPTED)
def test_load_instance(self, graph):
tag = factories.TagFactory()
assert graph.create(IDObfuscator.encode(tag), 'tag', {}).instance == tag
def id(self):
return (self.instance and IDObfuscator.encode(self.instance)) or self._id
def get_url(self, obj, view_name, request, format):
obfuscated_id = IDObfuscator.encode(obj)
kwargs = {self.lookup_url_kwarg: obfuscated_id}
return self.reverse(view_name, kwargs=kwargs, request=request, format=format)
def test_change_no_diff(self, graph):
tag = factories.TagFactory()
assert graph.create(IDObfuscator.encode(tag), 'tag', {'name': tag.name}).change == {}
def test_change_diff(self, graph):
tag = factories.TagFactory(name='tag1')
assert graph.create(IDObfuscator.encode(tag), 'tag', {'name': 'tag2'}).change == {'name': 'tag2'}
def test_change_datetime_no_change(self, graph):
work = factories.AbstractCreativeWorkFactory()
assert graph.create(IDObfuscator.encode(work), work._meta.model_name, {'date_updated': work.date_updated.isoformat()}).change == {}
def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(IDObfuscator.encode(tag), 'tag', {'date_updated': pendulum.fromtimestamp(0).isoformat()}).change == {'date_updated': pendulum.fromtimestamp(0).isoformat()}
def test_list_identifiers(self, post, all_about_anteaters):
parsed = oai_request({'verb': 'ListIdentifiers', 'metadataPrefix': 'oai_dc'}, post)
identifiers = parsed.xpath('//ns0:ListIdentifiers/ns0:header/ns0:identifier', namespaces=NAMESPACES)
assert len(identifiers) == 1
assert identifiers[0].text == 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
def test_oai_errors(self, verb, params, errors, post, all_about_anteaters):
ant_id = IDObfuscator.encode(all_about_anteaters)
data = {'verb': verb, **{k: v.format(id=ant_id) for k, v in params.items()}}
actual_errors = oai_request(data, post, errors=True)
for error_code in errors:
assert any(e.attrib.get('code') == error_code for e in actual_errors)
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data, context={'request': request})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job
).order_by('-created_at').values_list('id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response({
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task': async_result.id,
'ingest_job': request.build_absolute_uri(reverse('api:ingestjob-detail', args=[IDObfuscator.encode(ingester.job)])),
}
}, status=status.HTTP_202_ACCEPTED)
def serialize_tag(self, tag):
return {
'id': IDObfuscator.encode(tag),
'type': 'tag',
'name': safe_substr(tag.name),
}
