def test_esdumper_with_edtfext(app, db, minimal_record, location, date,
expected_start, expected_end):
"""Test edft extension implementation."""
# Create a simple extension that adds a computed field.
dumper = ElasticsearchDumper(extensions=[
EDTFDumperExt("metadata.publication_date"),
EDTFListDumperExt("metadata.dates", "date"),
])
minimal_record["metadata"]["publication_date"] = date
minimal_record["metadata"]["dates"] = [{"date": date}]
# Create the record
record = RDMRecord.create(minimal_record, parent=RDMParent.create({}))
db.session.commit()
# Dump it
dump = record.dumps(dumper=dumper)
assert dump["metadata"]["publication_date_range"]["gte"] == expected_start
assert dump["metadata"]["publication_date_range"]["lte"] == expected_end
assert dump["metadata"]["publication_date"] == date
assert dump["metadata"]["dates"][0]["date_range"]["gte"] == expected_start
assert dump["metadata"]["dates"][0]["date_range"]["lte"] == expected_end
assert dump["metadata"]["dates"][0]["date"] == date
# Load it
new_record = RDMRecord.loads(dump, loader=dumper)
assert "publication_date_range" not in new_record["metadata"]
assert "publication_date" in new_record["metadata"]
assert "date_range" not in new_record["metadata"]["dates"][0]
assert "date" in new_record["metadata"]["dates"][0]
def test_eslistdumper_with_edtfext_parse_error(app, db, minimal_record):
"""Test edft extension implementation."""
dumper = ElasticsearchDumper(
extensions=[
EDTFListDumperExt("metadata.creators", "family_name"),
]
)
# Create the record
record = RDMRecord.create(minimal_record, parent=RDMParent.create({}))
db.session.commit()
# Dump it
dump = record.dumps(dumper=dumper)
person_or_org = dump["metadata"]["creators"][0]["person_or_org"]
assert "family_name_range" not in person_or_org
assert "family_name" in person_or_org
# Load it
new_record = RDMRecord.loads(dump, loader=dumper)
person_or_org = dump["metadata"]["creators"][0]["person_or_org"]
assert 'family_name_range' not in person_or_org
assert 'family_name' in person_or_org
assert 'type_start' not in new_record['metadata']['resource_type']
assert 'type_end' not in new_record['metadata']['resource_type']
assert 'type' in new_record['metadata']['resource_type']
def test_doi_publish_versions(app, location, minimal_record, identity_simple,
mocker):
client = mocker.patch(
"invenio_rdm_records.services.pids.providers.datacite."
"DOIDataCiteClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCiteRESTClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCite43JSONSerializer")
component = ExternalPIDsComponent(current_rdm_records.records_service)
doi_provider = DOIDataCitePIDProvider(client)
data = minimal_record.copy()
# make sure `pids` field is added on create
del data["pids"]
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == {}
# publish
record = RDMRecord.publish(draft)
# NOTE: simulate metadata component required by datacite serialization
record["metadata"] = draft["metadata"]
component.publish(identity_simple, draft=draft, record=record)
doi_v1 = record.pids["doi"]
# DOI
assert doi_v1
assert doi_v1["identifier"]
assert doi_v1["provider"] == "datacite"
assert doi_v1["client"] == "datacite" # Default since no values given
doi_pid = doi_provider.get(doi_v1["identifier"])
assert doi_pid.object_uuid == record.id
# create a new version (v2) of the record
draft_v2 = RDMDraft.new_version(record)
component.new_version(identity_simple, draft=draft_v2, record=record)
assert "pids" in draft and draft.pids == {}
# publish v2
record_v2 = RDMRecord.publish(draft_v2)
# NOTE: simulate metadata component required by datacite serialization
draft_v2["metadata"] = minimal_record["metadata"]
component.publish(identity_simple, draft=draft_v2, record=record_v2)
doi_v2 = record_v2.pids["doi"]
# DOI
assert doi_v2
assert doi_v2["identifier"]
assert doi_v2["provider"] == "datacite"
assert doi_v2["client"] == "datacite" # Default since no values given
assert doi_v1 != doi_v2
doi_v2_pid = doi_provider.get(doi_v2["identifier"])
assert doi_v2_pid.object_uuid == record_v2.id
assert doi_v1 != doi_v2
def test_locationsdumper_with_polygon_and_mock_shapely(app, db,
minimal_record):
with unittest.mock.patch(
'invenio_rdm_records.records.dumpers.locations.shapely'
) as shapely:
dumper = ElasticsearchDumper(extensions=[LocationsDumper()])
minimal_record['metadata']['locations'] = {
'features': [{
'geometry': {
'type':
'Polygon',
'coordinates': [[
[100.0, 0.0],
[101.0, 0.0],
[101.0, 1.0],
[100.0, 1.0],
[100.0, 0.0],
]]
}
}],
}
record = RDMRecord.create(minimal_record)
shape = unittest.mock.Mock()
shape.centroid.x, shape.centroid.y = 100.5, 0.5
shapely.geometry.shape.return_value = shape
dump = record.dumps(dumper=dumper)
shapely.geometry.shape.assert_called_once_with(
minimal_record['metadata']['locations']['features'][0]['geometry'])
assert dump['metadata']['locations']['features'][0]['centroid'] == \
[100.5, 0.5]
def test_unmanaged_duplicated_doi(not_req_unmanaged_pid_cmp, minimal_record,
identity_simple, location):
component = not_req_unmanaged_pid_cmp
pids = {
"nrequman": {
"identifier": "avalues",
"provider": "nrequman",
}
}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == pids
# publish
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
assert record.pids == pids
# create a second record with the same unmanaged DOI
data = minimal_record.copy()
data["pids"] = pids
draft = RDMDraft.create(data)
with pytest.raises(ValidationError):
component.create(identity_simple, data=data, record=draft)
def test_create_managed_doi_with_no_value(req_managed_pid_cmp, minimal_record,
identity_simple, mocker, location):
client = mocker.patch(
"invenio_rdm_records.services.pids.providers.datacite."
"DOIDataCiteClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCiteRESTClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCite43JSONSerializer")
component = req_managed_pid_cmp
pids = {"doi": {"provider": "datacite", "client": "rdm"}}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == pids
# publish
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
doi = record.pids.get("doi")
assert doi
assert doi["identifier"]
assert doi["provider"] == "datacite"
assert doi["client"] == "rdm" # tests non-default client works
provider = DOIDataCitePIDProvider(client)
assert provider.get(doi["identifier"]).is_registered()
def test_unmanaged_required_pid_value(req_pid_unmanaged_cmp, minimal_record,
identity_simple, location):
component = req_pid_unmanaged_cmp
pids = {
"requman": {
"identifier": "value",
"provider": "requman",
}
}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == pids
# publish
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
assert record.pids == pids
provider = RequiredUnmanagedPIDProvider("requir")
pid = provider.get_by_record(record.id, pid_value="value")
assert pid.is_registered()
def test_create_managed_doi_empty_pids(req_managed_pid_cmp, minimal_record,
identity_simple, mocker, location):
client = mocker.patch(
"invenio_rdm_records.services.pids.providers.datacite."
"DOIDataCiteClient")
mocker.patch("invenio_rdm_records.services.config.DOIDataCiteClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCiteRESTClient")
component = req_managed_pid_cmp
pids = {}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == pids
# publish
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
doi = record.pids.get("doi")
assert doi
assert doi["identifier"]
assert doi["provider"] == "datacite"
assert doi["client"] == "datacite" # default
provider = DOIDataCitePIDProvider(client)
assert provider.get_by_record(record.id, pid_value=doi["identifier"])
def test_unmanaged_no_required_no_partial_value(not_req_unmanaged_pid_cmp,
minimal_record,
identity_simple, location):
component = not_req_unmanaged_pid_cmp
# if no name provided, one of the configured must be required
# since is not required we need to pass the provider name
pids = {
"nrequman": {
"provider": "nrequman",
}
}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == pids
# publish
# NOTE: Better fail than strip and not tell the user.
record = RDMRecord.publish(draft)
with pytest.raises(ValidationError):
component.publish(identity_simple, draft=draft, record=record)
def test_create_managed_doi_with_value(req_managed_pid_cmp, minimal_record,
identity_simple, mocker, location):
client = mocker.patch(
"invenio_rdm_records.services.pids.providers.datacite."
"DOIDataCiteClient")
mocker.patch("invenio_rdm_records.services.pids.providers.datacite." +
"DataCiteRESTClient")
component = req_managed_pid_cmp
# create a minimal draft
data = minimal_record.copy()
draft = RDMDraft.create(data)
component.create(identity_simple, data=data, record=draft)
assert "pids" in draft and draft.pids == {}
# when provider is managed and required, and the identifier is in the
# payload, it means that it was reserved via the `reserve` endpoint
provider = DOIDataCitePIDProvider(client)
pid = provider.create(draft)
provider.reserve(pid, draft)
pids = {
"doi": {
"identifier": pid.pid_value,
"provider": "datacite",
"client": "rdm"
}
}
draft["pids"] = pids
# publish
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
assert record.pids == pids
assert provider.get(pid.pid_value).is_registered()
def test_access_component_valid(minimal_record, parent, identity_simple,
users):
record = RDMRecord.create(minimal_record, parent=parent)
component = AccessComponent(current_rdm_records.records_service)
component.create(identity_simple, minimal_record, record)
assert len(record.parent.access.owners) > 0
def test_locationsdumper_with_polygon_and_no_shapely(app, db, minimal_record):
dumper = ElasticsearchDumper(extensions=[LocationsDumper()])
minimal_record['metadata']['locations'] = {
'features': [{
'geometry': {
'type':
'Polygon',
'coordinates': [[
[100.0, 0.0],
[101.0, 0.0],
[101.0, 1.0],
[100.0, 1.0],
[100.0, 0.0],
]]
}
}],
}
record = RDMRecord.create(minimal_record)
with pytest.warns(UserWarning):
dump = record.dumps(dumper=dumper)
assert 'centroid' not in dump['metadata']['locations']['features'][0]
def test_locationsdumper_with_no_featurecollection(app, db, minimal_record):
dumper = ElasticsearchDumper(extensions=[LocationsDumper()])
record = RDMRecord.create(minimal_record)
# Dump it
dump = record.dumps(dumper=dumper)
def test_locationsdumper_with_point_geometry(app, db, minimal_record):
dumper = ElasticsearchDumper(extensions=[LocationsDumper()])
minimal_record['metadata']['locations'] = {
'features': [{
'geometry': {
'type': 'Point',
'coordinates': [6.052778, 46.234167]
}
}]
}
record = RDMRecord.create(minimal_record)
# Dump it
dump = record.dumps(dumper=dumper)
# Centroid has been inferred
dumped_feature = dump['metadata']['locations']['features'][0]
expected_feature = minimal_record['metadata']['locations']['features'][0]
assert (dumped_feature['centroid'] == expected_feature['geometry']
['coordinates'])
# And it round-trips
assert (record.loads(dump, loader=dumper)['metadata']['locations'] ==
minimal_record['metadata']['locations'])
def test_locationsdumper_with_polygon_and_shapely(app, db, minimal_record):
pytest.importorskip('shapely')
dumper = ElasticsearchDumper(extensions=[LocationsDumper()])
# This also tests shapes with elevations
minimal_record['locations'] = {
'features': [{
'geometry': {
'type':
'Polygon',
'coordinates': [[
[100.0, 0.0, 10],
[101.0, 0.0, 10],
[101.0, 1.0, 30],
[100.0, 1.0, 30],
[100.0, 0.0, 10],
]]
}
}],
}
record = RDMRecord.create(minimal_record)
dump = record.dumps(dumper=dumper)
# 3D geometries still lead to 2D centroids
assert dump['locations']['features'][0]['centroid'] == [100.5, 0.5]
def test_permission_policy_generators(app, anyuser_identity,
authenticated_identity,
superuser_identity,
system_process_identity):
"""Test permission policies with given Identities."""
policy = TestRDMPermissionPolicy
# TODO: add to fixture
rest_record = RDMRecord.create({}, access={}, parent=RDMParent.create({}))
rest_record.access.protection.set("restricted", "restricted")
rest_record.parent.access.owners.add({'user': 1})
# TODO: add to fixture
pub_record = RDMRecord.create({}, access={}, parent=RDMParent.create({}))
pub_record.access.protection.set("public", "public")
pub_record.parent.access.owners.add({'user': 21})
assert policy(action='search').allows(anyuser_identity)
assert policy(action='search').allows(system_process_identity)
assert policy(action='create').allows(authenticated_identity)
assert policy(action='create').allows(system_process_identity)
assert isinstance(policy(action='update').generators[0], Disable)
assert isinstance(policy(action='delete').generators[0], Disable)
assert policy(action='read').generators[0].needs(record=rest_record) == {
UserNeed(1)
}
assert policy(action='read').generators[0].needs(record=pub_record) == {
system_process, any_user
}
assert policy(action='read_files').generators[0].needs(
record=rest_record) == {UserNeed(1)}
assert isinstance(policy(action='update_files').generators[0], Disable)
assert policy(action='read_draft').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='update_draft').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='delete_draft').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='read_draft_files').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='read_update_files').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='publish').generators[0].needs(
record=rest_record) == [UserNeed(1)]
assert policy(action='manage').generators[0].needs(record=rest_record) == [
UserNeed(1)
]
def test_access_component_unknown_owner(minimal_record, parent,
identity_simple, users):
record = RDMRecord.create(minimal_record, parent=parent)
record.parent["access"]["owned_by"] = [{"user": -1337}]
component = AccessComponent(RDMRecordService())
# both should work, since 'access.owned_by' is ignored for anybody
# other than system processes
component.create(identity_simple, minimal_record, record)
component.update_draft(identity_simple, minimal_record, record)
def _create_record():
"""Create a record."""
# create draft
draft = RDMDraft.create(minimal_record)
draft.commit()
db.session.commit()
# publish and get record
record = RDMRecord.publish(draft)
record.commit()
db.session.commit()
return record
def test_access_field_clear_embargo(minimal_record, parent):
next_year = arrow.utcnow().datetime + timedelta(days=+365)
minimal_record["access"]["embargo"] = {
"until": next_year.strftime("%Y-%m-%d"),
"active": True,
"reason": "nothing in particular",
}
rec = RDMRecord.create(minimal_record, parent=parent)
rec.access.embargo.clear()
assert not rec.access.embargo
def test_access_component_unknown_owner_with_system_process(
minimal_record, parent, users):
record = RDMRecord.create(minimal_record, parent=parent)
record.parent["access"]["owned_by"] = [{"user": -1337}]
component = AccessComponent(current_rdm_records.records_service)
with pytest.raises(ValidationError):
component.create(system_identity, minimal_record, record)
with pytest.raises(ValidationError):
component.update_draft(system_identity, minimal_record, record)
def test_esdumper_with_edtfext_parse_error(app, db, location, minimal_record):
"""Test edft extension implementation."""
# NOTE: We cannot trigger this on publication_date because it is checked
# by marshmallow on record creation. We can simply give a non date field.
dumper = ElasticsearchDumper(extensions=[
EDTFDumperExt("metadata.resource_type.type"),
])
# Create the record
record = RDMRecord.create(minimal_record, parent=RDMParent.create({}))
db.session.commit()
# Dump it
dump = record.dumps(dumper=dumper)
assert "type_range" not in dump["metadata"]["resource_type"]
assert "type" in dump["metadata"]["resource_type"]
# Load it
new_record = RDMRecord.loads(dump, loader=dumper)
assert "type_range" not in new_record["metadata"]["resource_type"]
assert "type" in new_record["metadata"]["resource_type"]
def test_eslistdumper_with_edtfext_not_defined(app, db, minimal_record):
"""Test edft extension implementation."""
# Create a simple extension that adds a computed field.
dumper = ElasticsearchDumper(extensions=[
EDTFListDumperExt("metadata.non_existing_array_field", "date"),
])
# Create the record
record = RDMRecord.create(minimal_record, parent=RDMParent.create({}))
db.session.commit()
# Dump it
dump = record.dumps(dumper=dumper)
assert "non_existing_array_field_range" not in dump["metadata"]
assert "non_existing_array_field" not in dump["metadata"]
# Load it
new_record = RDMRecord.loads(dump, loader=dumper)
assert "non_existing_array_field_range" not in new_record["metadata"]
assert "non_existing_array_field" not in new_record["metadata"]
def test_access_component_set_owner(minimal_record, parent, users):
user = users[0]
identity = Identity(user.id)
identity.provides.add(UserNeed(user.id))
record = RDMRecord.create(minimal_record, parent=parent)
record.parent["access"]["owned_by"] = []
component = AccessComponent(current_rdm_records.records_service)
component.create(identity, minimal_record, record)
assert len(record.access.owners) == 1
assert list(record.access.owners)[0].resolve() == user
def test_access_field_update_protection(minimal_record, parent, users):
minimal_record["access"]["record"] = "restricted"
minimal_record["access"]["files"] = "restricted"
rec = RDMRecord.create(minimal_record, parent=parent)
assert rec.access.protection.record == "restricted"
assert rec.access.protection.files == "restricted"
rec.access.protection.set("public", "public")
rec.commit()
assert rec["access"]["record"] == "public"
assert rec["access"]["files"] == "public"
def _owned_record():
data = {
"access": {
"owned_by": [
{"user": 16},
{"user": 17},
]
}
}
record = RDMRecord.create({}, access={})
record.parent = RDMParent(data)
return record
def test_access_field_on_record(running_app, minimal_record, parent, users):
next_year = arrow.utcnow().datetime + timedelta(days=+365)
minimal_record["access"]["embargo"] = {
"until": next_year.strftime("%Y-%m-%d"),
"active": True,
"reason": "nothing in particular",
}
rec = RDMRecord.create(minimal_record, parent=parent)
assert isinstance(rec.access, RecordAccess)
assert isinstance(rec.access.protection, Protection)
assert rec.access.protection.record == minimal_record["access"]["record"]
assert rec.access.protection.files == minimal_record["access"]["files"]
assert isinstance(rec.access.embargo, Embargo)
def test_esdumper_with_externalpidsext(app, db, minimal_record, location):
# Create a simple extension that adds a computed field.
dumper = ElasticsearchDumper(
extensions=[PIDsDumperExt()]
)
minimal_record["pids"] = {
"doi": {
"identifier": "10.5281/zenodo.1234",
"provider": "datacite",
"client": "zenodo"
},
"handle": {
"identifier": "9.12314",
"provider": "cern-handle",
"client": "zenodo"
}
}
# Create the record
record = RDMRecord.create(minimal_record, parent=RDMParent.create({}))
db.session.commit()
# Dump it
dump = record.dumps(dumper=dumper)
dumped_pids = dump["pids"]
for dumped_pid in dumped_pids:
pid_attrs = dumped_pid.keys()
assert "scheme" in pid_attrs
assert "identifier" in pid_attrs
assert "provider" in pid_attrs
assert "client" in pid_attrs
# Load it
new_record = RDMRecord.loads(dump, loader=dumper)
assert minimal_record["pids"] == new_record["pids"]
def test_publish_no_pids(no_pids_cmp, minimal_record, identity_simple,
location):
component = no_pids_cmp
# empty pids field
pids = {}
# make sure `pids` field is added
data = minimal_record.copy()
data["pids"] = pids
# create a minimal draft
draft = RDMDraft.create(data)
# no validation needed since /publish gets no payload (new data)
record = RDMRecord.publish(draft)
component.publish(identity_simple, draft=draft, record=record)
assert record.get("pids") == {}
def test_access_component_unknown_grant_subject(minimal_record, parent,
identity_simple, users):
record = RDMRecord.create(minimal_record, parent=parent)
record.parent["access"]["grants"] = [{
"subject": "user",
"id": "-1337",
"level": "view"
}]
component = AccessComponent(current_rdm_records.records_service)
with pytest.raises(ValidationError):
component.create(identity_simple, minimal_record, record)
with pytest.raises(ValidationError):
component.update_draft(identity_simple, minimal_record, record)
def test_access_field_update_embargo(minimal_record, parent, users):
next_year = arrow.utcnow().datetime + timedelta(days=+365)
minimal_record["access"]["embargo"] = {
"until": next_year.strftime("%Y-%m-%d"),
"active": True,
"reason": "nothing in particular",
}
rec = RDMRecord.create(minimal_record.copy(), parent=parent)
assert rec.access.embargo
rec.access.embargo.active = False
rec.access.embargo.reason = "can't remember"
rec.commit()
minimal_record["access"]["embargo"]["active"] = False
minimal_record["access"]["embargo"]["reason"] = "can't remember"
