def test_no_icon(self):
x = factories.AbstractCreativeWorkFactory()
source = factories.SourceFactory(icon='')
x.sources.add(source.user)
# Have to use % formats because of {}s everywhere
result = schema.execute('''
query {
creativeWork(id: "%s") {
id,
title,
description,
sources {
title
}
}
}
''' % (IDObfuscator.encode(x), ))
assert not result.errors
assert result.data == OrderedDict([
('creativeWork', OrderedDict([
('id', IDObfuscator.encode(x)),
('title', x.title),
('description', x.description),
('sources', [])
]))
])
def test_no_icon(self, schema):
x = factories.AbstractCreativeWorkFactory()
source = factories.SourceFactory(icon='')
x.sources.add(source.user)
# Have to use % formats because of {}s everywhere
result = schema.execute('''
query {
creativeWork(id: "%s") {
id,
title,
description,
sources {
title
}
}
}
''' % (IDObfuscator.encode(x), ))
assert not result.errors
assert result.data == OrderedDict([
('creativeWork', OrderedDict([
('id', IDObfuscator.encode(x)),
('title', x.title),
('description', x.description),
('sources', [])
]))
])
def test_by_id(self, client):
nd = factories.NormalizedDataFactory(data={'@graph': []})
resp = client.get('/api/v2/normalizeddata/{}/'.format(IDObfuscator.encode(nd)))
assert resp.status_code == 200
assert resp.json()['data']['id'] == IDObfuscator.encode(nd)
assert resp.json()['data']['type'] == 'NormalizedData'
assert resp.json()['data']['attributes']['data'] == {'@graph': []}
def test_update_dependencies_accept(self, john_doe, normalized_data_id):
graph = ChangeGraph([{
'@id': IDObfuscator.encode(john_doe),
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {
'@id': IDObfuscator.encode(john_doe),
'@type': 'person'
},
'creative_work': {
'@id': '_:789',
'@type': 'preprint'
},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = models.ChangeSet.objects.from_graph(
graph, normalized_data_id)
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(
agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(
agent_relations__agent=john_doe).first().title == 'All About Cats'
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data,
context={
'request': request
})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(
request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job).order_by('-created_at').values_list(
'id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response(
{
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task':
async_result.id,
'ingest_job':
request.build_absolute_uri(
reverse('api:ingestjob-detail',
args=[IDObfuscator.encode(ingester.job)])),
}
},
status=status.HTTP_202_ACCEPTED)
def test_update_dependencies_accept(self, john_doe, normalized_data_id):
graph = ChangeGraph([{
'@id': IDObfuscator.encode(john_doe),
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {'@id': IDObfuscator.encode(john_doe), '@type': 'person'},
'creative_work': {'@id': '_:789', '@type': 'preprint'},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = models.ChangeSet.objects.from_graph(graph, normalized_data_id)
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).first().title == 'All About Cats'
def oai_identifier(self, work):
if isinstance(work, int):
share_id = IDObfuscator.encode_id(work, AbstractCreativeWork)
else:
share_id = IDObfuscator.encode(work)
return 'oai{delim}{repository}{delim}{id}'.format(
id=share_id,
repository=self.REPOSITORY_IDENTIFIER,
delim=self.IDENTIFER_DELIMITER)
def test_by_id(self, client):
source = Source.objects.first()
resp = client.get('{}{}/'.format(self.endpoint, IDObfuscator.encode(source)))
assert resp.status_code == 200
assert IDObfuscator.load(resp.json()['data']['id']) == source
assert resp.json()['data']['type'] == 'Source'
assert resp.json()['data']['attributes'] == {
'name': source.name,
'icon': 'http://testserver{}'.format(source.icon.url),
'homePage': source.home_page,
'longTitle': source.long_title,
}
def test_by_id(self, client):
source = Source.objects.first()
resp = client.get('{}{}/'.format(self.endpoint, IDObfuscator.encode(source)))
assert resp.status_code == 200
assert IDObfuscator.load(resp.json()['data']['id']) == source
assert resp.json()['data']['type'] == 'Source'
assert resp.json()['data']['attributes'] == {
'name': source.name,
'icon': 'http://testserver{}'.format(source.icon.url),
'homePage': source.home_page,
'longTitle': source.long_title,
}
def test_can_delete_work(self, client, normalized_data_id):
preprint = factories.AbstractCreativeWorkFactory(is_deleted=False)
preprint.administrative_change(type='share.dataset')
assert preprint.is_deleted is False
encoded_id = IDObfuscator.encode(preprint)
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 200
preprint.administrative_change(is_deleted=True)
assert preprint.is_deleted is True
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 403
assert response.json() == {
"errors": [{
"source": {
"pointer": "/data"
},
"detail": "This data set has been removed.",
"status": "403"
}]
}
response = client.get('/api/v2/datasets/')
assert response.status_code == 200
assert response.json() == {
'data': [],
'links': {
'next': None,
'prev': None
}
}
def initial(self, request, *args, **kwargs):
ret = super().initial(request, *args, **kwargs)
lookup_key = self.lookup_url_kwarg or self.lookup_field
if lookup_key not in self.kwargs:
return ret
lookup = self.kwargs[lookup_key]
expected_model = self.get_serializer_class().Meta.model
# Decode id's before the all the internal dispatching
try:
model, id = IDObfuscator().decode(lookup)
except InvalidID:
if re.fullmatch(r'\d+', lookup):
# Allow primary keys for debugging convenience.
# Mabye remove if people start abusing this to harvest by sequential PK.
return ret
raise serializers.ValidationError('Invalid ID')
if not issubclass(expected_model, model):
raise serializers.ValidationError('Invalid ID')
self.kwargs[lookup_key] = str(id)
return ret
def test_add_multiple_sources(self, celery_app):
source1 = factories.SourceFactory()
source2 = factories.SourceFactory()
work = factories.AbstractCreativeWorkFactory(title='All about Canada')
data = {
'@id': IDObfuscator.encode(work),
'@type': 'creativework',
'title': 'All aboot Canada'
}
nd1 = NormalizedData.objects.create(source=source1.user,
data={'@graph': [data]})
nd2 = NormalizedData.objects.create(source=source2.user,
data={'@graph': [data]})
assert work.sources.count() == 0
celery_app.tasks['share.tasks.disambiguate'](nd1.id)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 1
celery_app.tasks['share.tasks.disambiguate'](nd2.id)
work.refresh_from_db()
assert work.title == 'All aboot Canada'
assert work.sources.count() == 2
def test_successful_post_no_home_page(self, client, source_add_user,
mock_icon_urls):
test_data = get_post_body()
resp = client.post(
self.endpoint,
json.dumps(test_data),
content_type='application/vnd.api+json',
HTTP_AUTHORIZATION=source_add_user.authorization(),
)
assert resp.status_code == 201
data = flatten_write_response(resp)
created_label = data['source']['longTitle'].replace(' ', '_').lower()
created_user = ShareUser.objects.get(
pk=IDObfuscator.decode_id(data['user']['id']))
assert data['source']['longTitle'] == test_data['data']['attributes'][
'long_title']
assert data['source']['name'] == created_label
assert data['source']['homePage'] is None
assert data['user']['username'] == created_label
assert data['user']['token'] == created_user.accesstoken_set.first(
).token
assert created_user.is_trusted is True
assert data['sourceConfig']['label'] == created_label
def test_list_with_items(self, client):
user = ShareUserFactory()
banner = SiteBanner.objects.create(
title='Why wasnt I there',
description='I could have saved them',
created_by=user,
last_modified_by=user,
)
resp = client.get('/api/v2/site_banners/')
assert resp.status_code == 200
assert resp.json() == {
'data': [{
'id': IDObfuscator.encode(banner),
'type': 'SiteBanner',
'attributes': {
'color': 'info',
'icon': 'exclamation',
'title': 'Why wasnt I there',
'description': 'I could have saved them',
}
}],
'links': {
'first': 'http://testserver/api/v2/site_banners/?page=1',
'last': 'http://testserver/api/v2/site_banners/?page=1',
'next': None,
'prev': None,
},
'meta': {
'pagination': {'count': 1, 'pages': 1, 'page': 1},
}
}
def get_url(self, obj, view_name, request, format):
obfuscated_id = IDObfuscator.encode(obj)
kwargs = {self.lookup_url_kwarg: obfuscated_id}
return self.reverse(view_name,
kwargs=kwargs,
request=request,
format=format)
def test_list_with_items(self, client):
user = ShareUserFactory()
banner = SiteBanner.objects.create(
title='Why wasnt I there',
description='I could have saved them',
created_by=user,
last_modified_by=user,
)
resp = client.get('/api/v2/site_banners/')
assert resp.status_code == 200
assert resp.json() == {
'data': [{
'id': IDObfuscator.encode(banner),
'type': 'SiteBanner',
'attributes': {
'color': 'info',
'icon': 'exclamation',
'title': 'Why wasnt I there',
'description': 'I could have saved them',
}
}],
'links': {
'first': 'http://testserver/api/v2/site_banners/?page=1',
'last': 'http://testserver/api/v2/site_banners/?page=1',
'next': None,
'prev': None,
},
'meta': {
'pagination': {
'count': 1,
'pages': 1,
'page': 1
},
}
}
def test_change_diff(self, graph):
tag = factories.TagFactory(name='tag1')
assert graph.create(IDObfuscator.encode(tag), 'tag', {
'name': 'tag2'
}).change == {
'name': 'tag2'
}
def test_update_dependencies_accept(self, john_doe, normalized_data):
john_doe_id = IDObfuscator.encode(john_doe)
graph = MutableGraph.from_jsonld([{
'@id': john_doe_id,
'@type': 'person',
'given_name': 'Jane',
}, {
'@id': '_:456',
'@type': 'Creator',
'agent': {'@id': john_doe_id, '@type': 'person'},
'creative_work': {'@id': '_:789', '@type': 'preprint'},
}, {
'@id': '_:789',
'@type': 'preprint',
'title': 'All About Cats',
}])
change_set = ChangeSetBuilder(graph, normalized_data, matches={
john_doe_id: john_doe,
}).build_change_set()
change_set.accept()
john_doe.refresh_from_db()
assert john_doe.given_name == 'Jane'
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).count() == 1
assert models.Preprint.objects.filter(agent_relations__agent=john_doe).first().title == 'All About Cats'
def get_object(self):
queryset = self.filter_queryset(self.get_queryset(False))
# Perform the lookup filtering.
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
assert lookup_url_kwarg in self.kwargs, (
'Expected view %s to be called with a URL keyword argument '
'named "%s". Fix your URL conf, or set the `.lookup_field` '
'attribute on the view correctly.' %
(self.__class__.__name__, lookup_url_kwarg))
try:
(model,
decoded_pk) = IDObfuscator.decode(self.kwargs[lookup_url_kwarg])
concrete_model = self.serializer_class.Meta.model._meta.concrete_model
if model is not concrete_model:
raise serializers.ValidationError(
'The specified ID refers to an {}. Expected {}'.format(
model._meta.model_name,
concrete_model._meta.model_name))
except InvalidID:
raise serializers.ValidationError('Invalid ID')
filter_kwargs = {self.lookup_field: decoded_pk}
obj = get_object_or_404(queryset, **filter_kwargs)
# May raise a permission denied
self.check_object_permissions(self.request, obj)
return obj
def get_object(self):
queryset = self.filter_queryset(self.get_queryset())
# Perform the lookup filtering.
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
assert lookup_url_kwarg in self.kwargs, (
'Expected view %s to be called with a URL keyword argument '
'named "%s". Fix your URL conf, or set the `.lookup_field` '
'attribute on the view correctly.' %
(self.__class__.__name__, lookup_url_kwarg)
)
try:
(model, decoded_pk) = IDObfuscator.decode(self.kwargs[lookup_url_kwarg])
concrete_model = self.serializer_class.Meta.model._meta.concrete_model
if model is not concrete_model:
raise serializers.ValidationError('The specified ID refers to an {}. Expected {}'.format(model._meta.model_name, concrete_model._meta.model_name))
except InvalidID:
raise serializers.ValidationError('Invalid ID')
filter_kwargs = {self.lookup_field: decoded_pk}
obj = get_object_or_404(queryset, **filter_kwargs)
# May raise a permission denied
self.check_object_permissions(self.request, obj)
return obj
def test_get_record(self, post, all_about_anteaters):
ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
parsed = oai_request({'verb': 'GetRecord', 'metadataPrefix': 'oai_dc', 'identifier': ant_id}, post)
records = parsed.xpath('//ns0:GetRecord/ns0:record', namespaces=NAMESPACES)
assert len(records) == 1
assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text
assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text
def assert_indexed(self):
encoded_id = IDObfuscator.encode(self.work)
doc = self.elastic.es_client.get(index=self.elastic.es_index,
doc_type='creativeworks',
id=encoded_id)
assert doc['found'] is True
assert doc['_id'] == encoded_id
return doc
def refs(n):
if isinstance(n, list):
return [refs(node) for node in n]
instance = self._get_match(n)
return {
'@id': IDObfuscator.encode(instance) if instance else n.id,
'@type': n.type,
}
def refs(n):
if isinstance(n, list):
return [refs(node) for node in n]
instance = self._get_match(n)
return {
'@id': IDObfuscator.encode(instance) if instance else n.id,
'@type': n.type,
}
def test_list_records(self, post, all_about_anteaters, django_assert_num_queries):
with django_assert_num_queries(2):
parsed = oai_request({'verb': 'ListRecords', 'metadataPrefix': 'oai_dc'}, post)
records = parsed.xpath('//ns0:ListRecords/ns0:record', namespaces=NAMESPACES)
assert len(records) == 1
assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text
ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text
def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(
IDObfuscator.encode(tag), 'tag', {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}).change == {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}
def initial_pass(self, nodes):
for node in nodes:
if str(node.id).startswith('_:'):
continue
try:
match = IDObfuscator.resolve(node.id)
self.add_match(node, match)
except InvalidID:
pass
def resolve_oai_identifier(self, identifier):
try:
splid = identifier.split(self.IDENTIFER_DELIMITER)
if len(splid) != 3 or splid[:2] != ['oai', self.REPOSITORY_IDENTIFIER]:
raise InvalidID(identifier)
return IDObfuscator.resolve(splid[-1])
except (AbstractCreativeWork.DoesNotExist, InvalidID):
self.errors.append(oai_errors.BadRecordID(identifier))
return None
def delete_works(self, queryset, source_config, dry_run):
for work in queryset:
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(
source_config.source.name, IDObfuscator.encode(work))),
work.title))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(
IDObfuscator.encode(work))), work.title))
if not dry_run:
work.administrative_change(is_deleted=True)
def test_ignore_generic_work_type(self, change_factory, all_about_anteaters):
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(all_about_anteaters),
'@type': 'creativework'
}]
}, disambiguate=True)
assert cs is None
def initial_pass(self, nodes):
for node in nodes:
if str(node.id).startswith('_:'):
continue
try:
match = IDObfuscator.resolve(node.id)
self.add_match(node, match)
except InvalidID:
pass
def resolve_oai_identifier(self, identifier):
try:
splid = identifier.split(self.IDENTIFER_DELIMITER)
if len(splid) != 3 or splid[:2] != ['oai', self.REPOSITORY_IDENTIFIER]:
raise InvalidID(identifier)
return IDObfuscator.resolve(splid[-1])
except (AbstractCreativeWork.DoesNotExist, InvalidID):
self.errors.append(oai_errors.BadRecordID(identifier))
return None
def assert_indexed(self):
encoded_id = IDObfuscator.encode(self.work)
doc = self.elastic.es_client.get(
index=self.elastic.es_index,
doc_type='creativeworks',
id=encoded_id
)
assert doc['found'] is True
assert doc['_id'] == encoded_id
return doc
def get_raws(obj):
if isinstance(obj, str):
model, id = IDObfuscator.decode(obj)
else:
model = obj._meta.model
id = obj.id
return RawDatum.objects.filter(
normalizeddata__changeset__changes__target_id=id,
normalizeddata__changeset__changes__target_type=ContentType.objects.
get_for_model(model, for_concrete_model=True))
def get_raws(obj):
if isinstance(obj, str):
model, id = IDObfuscator.decode(obj)
else:
model = obj._meta.model
id = obj.id
return RawData.objects.filter(
normalizeddata__changeset__changes__target_id=id,
normalizeddata__changeset__changes__target_type=ContentType.objects.get_for_model(model, for_concrete_model=True)
)
def populate_types(data):
model = apps.get_model(data['type'])
data['id'] = IDObfuscator.encode_id(data['id'], model)
data['type'] = model._meta.verbose_name
data['types'] = []
for parent in model.mro():
if not parent._meta.proxy:
break
data['types'].append(parent._meta.verbose_name)
return data
def populate_types(data):
model = apps.get_model(data['type'])
data['id'] = IDObfuscator.encode_id(data['id'], model)
data['type'] = model._meta.verbose_name
data['types'] = []
for parent in model.mro():
if not parent._meta.proxy:
break
data['types'].append(parent._meta.verbose_name)
return data
def test_oai_errors(self, verb, params, errors, post, all_about_anteaters):
ant_id = IDObfuscator.encode(all_about_anteaters)
data = {
'verb': verb,
**{k: v.format(id=ant_id)
for k, v in params.items()}
}
actual_errors = oai_request(data, post, errors=True)
for error_code in errors:
assert any(
e.attrib.get('code') == error_code for e in actual_errors)
def delete_works(self, works_qs, source_config, dry_run, superfluous,
delete_related):
works_deleted = []
if not superfluous:
works_qs = works_qs.filter(is_deleted=False)
for work in works_qs.prefetch_related('sources'):
works_deleted.append(work.id)
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(
source_config.source.name, IDObfuscator.encode(work))),
work.title))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(
IDObfuscator.encode(work))), work.title))
if not dry_run:
work.administrative_change(is_deleted=True)
self.stdout.write('\nProcessed {} works!'.format(len(works_deleted)),
style_func=self.style.SUCCESS)
if not delete_related:
return
self.stdout.write('\nNow deleting related works...\n')
related_works = AbstractCreativeWork.objects.filter(
Q(incoming_creative_work_relations__subject_id__in=works_deleted) |
Q(outgoing_creative_work_relations__related_id__in=works_deleted),
is_deleted=False,
sources__id=source_config.source.user_id).prefetch_related(
'sources')
# Traverse related works only one level deep, please
self.delete_works(related_works, source_config, dry_run, superfluous,
False)
def test_list_identifiers(self, post, all_about_anteaters):
parsed = oai_request(
{
'verb': 'ListIdentifiers',
'metadataPrefix': 'oai_dc'
}, post)
identifiers = parsed.xpath(
'//ns0:ListIdentifiers/ns0:header/ns0:identifier',
namespaces=NAMESPACES)
assert len(identifiers) == 1
assert identifiers[0].text == 'oai:share.osf.io:{}'.format(
IDObfuscator.encode(all_about_anteaters))
def _resolve_ref(self, ref):
model = apps.get_model('share', model_name=ref['@type'])
ct = ContentType.objects.get_for_model(model, for_concrete_model=True)
try:
if ref['@id'].startswith('_:'):
return next(change.target for change in self._changes_cache
if change.target_type == ct
and change.node_id == ref['@id'] and change.target)
return model._meta.concrete_model.objects.get(
pk=IDObfuscator.decode_id(ref['@id']))
except (StopIteration, model.DoesNotExist) as ex:
raise Exception(
'Could not resolve reference {}'.format(ref)) from ex
def __init__(self, graph, id, type, attrs, namespace=None):
self.graph = graph
self._id = id
self._type = type.lower()
self.instance = None
self.attrs = attrs
self.extra = attrs.pop('extra', {})
self.context = attrs.pop('@context', {})
self.namespace = namespace
if not self.is_blank:
self.instance = IDObfuscator.load(self.id, None)
if not self.instance or self.instance._meta.concrete_model is not self.model._meta.concrete_model:
raise UnresolvableReference((self.id, self.type))
def __init__(self, graph, id, type, attrs, namespace=None):
self.graph = graph
self._id = id
self._type = type.lower()
self.instance = None
self.attrs = attrs
self.extra = attrs.pop('extra', {})
self.context = attrs.pop('@context', {})
self.namespace = namespace
if not self.is_blank:
self.instance = IDObfuscator.load(self.id, None)
if not self.instance or self.instance._meta.concrete_model is not self.model._meta.concrete_model:
raise UnresolvableReference((self.id, self.type))
def _resolve_ref(self, ref):
model = apps.get_model('share', model_name=ref['@type'])
ct = ContentType.objects.get_for_model(model, for_concrete_model=True)
try:
if ref['@id'].startswith('_:'):
return next(
change.target
for change in self._changes_cache
if change.target_type == ct
and change.node_id == ref['@id']
and change.target
)
return model._meta.concrete_model.objects.get(pk=IDObfuscator.decode_id(ref['@id']))
except (StopIteration, model.DoesNotExist) as ex:
raise Exception('Could not resolve reference {}'.format(ref)) from ex
def get_search_results(self, request, queryset, search_term):
try:
return queryset.filter(id=IDObfuscator.decode_id(search_term)), False
except InvalidID:
pass
# Overriden because there is no way to opt out of a case insensitive search
search_fields = self.get_search_fields(request)
use_distinct = bool(search_term)
if search_fields and search_term:
orm_lookups = ['{}__startswith'.format(search_field) for search_field in search_fields]
for bit in search_term.split():
or_queries = [models.Q(**{orm_lookup: bit}) for orm_lookup in orm_lookups]
queryset = queryset.filter(reduce(operator.or_, or_queries))
return queryset, use_distinct
def test_change_agent_type(self, change_factory, university_of_whales):
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(university_of_whales),
'@type': 'consortium'
}]
}, disambiguate=True)
assert models.Institution.objects.count() == 1
assert models.Consortium.objects.count() == 0
(org,) = cs.accept()
assert org.type == 'share.consortium'
assert org.id == university_of_whales.id
assert org.name == university_of_whales.name
assert models.Institution.objects.count() == 0
assert models.Consortium.objects.count() == 1
def test_work_type_stays_nongeneric(self, change_factory, all_about_anteaters):
new_title = 'Some about Anteaters'
cs = change_factory.from_graph({
'@graph': [{
'@id': IDObfuscator.encode(all_about_anteaters),
'@type': 'creativework',
'title': new_title
}]
}, disambiguate=True)
assert all_about_anteaters.type == 'share.article'
assert models.Publication.objects.count() == 1
cs.accept()
all_about_anteaters.refresh_from_db()
assert all_about_anteaters.type == 'share.article'
assert all_about_anteaters.title == new_title
def test_can_delete_work(self, client, normalized_data_id):
preprint = factories.AbstractCreativeWorkFactory(is_deleted=False)
preprint.administrative_change(type='share.dataset')
assert preprint.is_deleted is False
encoded_id = IDObfuscator.encode(preprint)
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 200
preprint.administrative_change(is_deleted=True)
assert preprint.is_deleted is True
response = client.get('/api/v2/datasets/{}/'.format(encoded_id))
assert response.status_code == 403
assert response.json() == {"errors": [{"source": {"pointer": "/data"}, "detail": "This data set has been removed.", "status": "403"}]}
response = client.get('/api/v2/datasets/')
assert response.status_code == 200
assert response.json() == {'data': [], 'links': {'next': None, 'prev': None}}
def test_successful_post_no_home_page(self, client, source_add_user, mock_icon_urls):
test_data = get_post_body()
resp = client.post(
self.endpoint,
json.dumps(test_data),
content_type='application/vnd.api+json',
HTTP_AUTHORIZATION=source_add_user.authorization(),
)
assert resp.status_code == 201
data = flatten_write_response(resp)
created_label = data['source']['longTitle'].replace(' ', '_').lower()
created_user = ShareUser.objects.get(pk=IDObfuscator.decode_id(data['user']['id']))
assert data['source']['longTitle'] == test_data['data']['attributes']['long_title']
assert data['source']['name'] == created_label
assert data['source']['homePage'] is None
assert data['user']['username'] == created_label
assert data['user']['token'] == created_user.accesstoken_set.first().token
assert created_user.is_trusted is True
def test_get_data(self, generator, model, route, controlled_values, client, Graph, ingest):
ingest(Graph(initial))
graph = Graph(*generator)
Regulator().regulate(graph)
matches = Matcher(DatabaseStrategy()).find_all_matches(graph)
for node in graph:
if node.type == model:
expected = node
expected_id = IDObfuscator.encode(matches[node])
break
response = client.get('/api/v2/{}/{}/'.format(route, expected_id))
actual = json.loads(response.content.decode(encoding='UTF-8'))
assert response.status_code == 200
assert actual['data']['id'] == expected_id
assert actual['data']['attributes']['type'] == expected.type
for value in controlled_values:
assert actual['data']['attributes'][value] == expected[camelCase_to_underscore(value)]
def delete_works(self, works_qs, source_config, dry_run, superfluous, delete_related):
works_deleted = []
if not superfluous:
works_qs = works_qs.filter(is_deleted=False)
for work in works_qs.prefetch_related('sources'):
works_deleted.append(work.id)
# If we've heard about the work from another source, just remove this source from it instead of deleting
if len(work.sources.all()) > 1:
self.stdout.write('{}: {}'.format(
self.style.WARNING('Removing {} from {}'.format(source_config.source.name, IDObfuscator.encode(work))),
work.title
))
if not dry_run:
work.sources.remove(source_config.source.user)
# poke it to reindex
work.administrative_change(allow_empty=True)
else:
self.stdout.write('{}: {}'.format(
self.style.NOTICE('Deleting work {}'.format(IDObfuscator.encode(work))),
work.title
))
if not dry_run:
work.administrative_change(is_deleted=True)
self.stdout.write('\nProcessed {} works!'.format(len(works_deleted)), style_func=self.style.SUCCESS)
if not delete_related:
return
self.stdout.write('\nNow deleting related works...\n')
related_works = AbstractCreativeWork.objects.filter(
Q(incoming_creative_work_relations__subject_id__in=works_deleted) | Q(outgoing_creative_work_relations__related_id__in=works_deleted),
is_deleted=False,
sources__id=source_config.source.user_id
).prefetch_related('sources')
# Traverse related works only one level deep, please
self.delete_works(related_works, source_config, dry_run, superfluous, False)
def create(self, request, *args, **kwargs):
serializer = self.get_serializer_class()(data=request.data, context={'request': request})
serializer.is_valid(raise_exception=True)
with transaction.atomic():
# Hack for back-compat: Ingest halfway synchronously, then apply changes asynchronously
ingester = Ingester(serializer.validated_data['data']).as_user(request.user).ingest(apply_changes=False)
ingester.job.reschedule(claim=True)
nd_id = models.NormalizedData.objects.filter(
raw=ingester.raw,
ingest_jobs=ingester.job
).order_by('-created_at').values_list('id', flat=True).first()
async_result = ingest.delay(job_id=ingester.job.id, urgent=True)
# TODO Use an actual serializer
return Response({
'id': IDObfuscator.encode_id(nd_id, models.NormalizedData),
'type': 'NormalizedData',
'attributes': {
'task': async_result.id,
'ingest_job': request.build_absolute_uri(reverse('api:ingestjob-detail', args=[IDObfuscator.encode(ingester.job)])),
}
}, status=status.HTTP_202_ACCEPTED)
def get_object(self, view_name, view_args, view_kwargs):
obfuscated_id = view_kwargs[self.lookup_url_kwarg]
return IDObfuscator.resolve(obfuscated_id)
def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(IDObfuscator.encode(tag), 'tag', {'date_updated': pendulum.fromtimestamp(0).isoformat()}).change == {'date_updated': pendulum.fromtimestamp(0).isoformat()}
def test_oai_errors(self, verb, params, errors, post, all_about_anteaters):
ant_id = IDObfuscator.encode(all_about_anteaters)
data = {'verb': verb, **{k: v.format(id=ant_id) for k, v in params.items()}}
actual_errors = oai_request(data, post, errors=True)
for error_code in errors:
assert any(e.attrib.get('code') == error_code for e in actual_errors)
def test_list_identifiers(self, post, all_about_anteaters):
parsed = oai_request({'verb': 'ListIdentifiers', 'metadataPrefix': 'oai_dc'}, post)
identifiers = parsed.xpath('//ns0:ListIdentifiers/ns0:header/ns0:identifier', namespaces=NAMESPACES)
assert len(identifiers) == 1
assert identifiers[0].text == 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters))
def get_url(self, obj, view_name, request, format):
obfuscated_id = IDObfuscator.encode(obj)
kwargs = {self.lookup_url_kwarg: obfuscated_id}
return self.reverse(view_name, kwargs=kwargs, request=request, format=format)
