def test_index_relations(app, db):
"""Test the index_relations method."""
data_v1 = {'body': u'test_body',
'title': u'test_title'}
data_v2 = {'body': u'test_body2',
'title': u'test_title2'}
# add first child to the relation
rec_v1 = Record.create(data_v1)
parent_pid = RecordIdProvider.create(object_type='rec',
object_uuid=None,
status=PIDStatus.REGISTERED).pid
data_v1['conceptrecid'] = parent_pid.pid_value
provider = RecordIdProvider.create('rec', rec_v1.id)
data_v1['recid'] = provider.pid.pid_value
versioning = PIDNodeVersioning(pid=parent_pid)
versioning.insert_child(child_pid=provider.pid)
db.session.commit()
output = index_relations(app, 'recid', record=rec_v1)
expected_output = \
{'relations': {
'version': [{
u'children': [{u'pid_type': u'recid',
u'pid_value': u'2'}],
u'index': 0,
u'is_child': True,
u'is_last': True,
u'is_parent': False,
u'next': None,
u'parent': {u'pid_type': u'recid',
u'pid_value': u'1'},
u'previous': None,
u'type': 'version'}]}}
assert compare_dictionaries(output, expected_output)
# add second child to the relation
rec_v2 = Record.create(data_v2)
data_v2['conceptrecid'] = parent_pid.pid_value
provider_v2 = RecordIdProvider.create('rec', rec_v2.id)
versioning.insert_child(child_pid=provider_v2.pid)
db.session.commit()
output = index_relations(app, 'recid', record=rec_v2)
expected_output = \
{'relations': {
'version': [{
u'children': [{u'pid_type': u'recid',
u'pid_value': u'2'},
{u'pid_type': u'recid',
u'pid_value': u'3'}],
u'index': 1,
u'is_child': True,
u'is_last': True,
u'is_parent': False,
u'next': None,
u'parent': {u'pid_type': u'recid',
u'pid_value': u'1'},
u'previous': {u'pid_type': u'recid',
u'pid_value': u'2'},
u'type': 'version'}]}}
assert compare_dictionaries(output, expected_output)
def test_recordid_provider(app, db):
"""Test record id provider."""
with app.app_context():
provider = RecordIdProvider.create()
assert provider.pid
assert provider.pid.pid_type == 'recid'
assert provider.pid.pid_value == '1'
assert provider.pid.pid_provider is None
assert provider.pid.status == PIDStatus.RESERVED
assert provider.pid.object_type is None
assert provider.pid.object_uuid is None
# Assign to object immediately
rec_uuid = uuid.uuid4()
provider = RecordIdProvider.create(object_type='rec',
object_uuid=rec_uuid)
assert provider.pid
assert provider.pid.pid_type == 'recid'
assert provider.pid.pid_value == '2'
assert provider.pid.pid_provider is None
assert provider.pid.status == PIDStatus.REGISTERED
assert provider.pid.object_type == 'rec'
assert provider.pid.object_uuid == rec_uuid
pytest.raises(AssertionError, RecordIdProvider.create, pid_value='3')
def construct_record(collection, metadata, creator_id, schema):
# Creating a uuid4
recid = uuid4()
# Creating a PID for the record
provider = RecordIdProvider.create(object_type='rec', object_uuid=recid)
pid = provider.pid.pid_value
data = {}
_deposit = {}
_metadata = metadata
_deposit['created_by'] = creator_id
# _deposit['id'] = recid.int
_deposit['pid'] = {
"revision_id": 0,
"type": "recid",
"value": pid
}
_deposit['status'] = "draft"
_deposit['pid_value'] = pid
_deposit['control_number'] = pid
if collection.parent and collection.parent.parent.name == 'CERNAnalysisPreservation':
data['experiment'] = collection.parent.name
data['collections'] = [collection.name]
data['$schema'] = schema
# data["recid"] = recid.int
data["_deposit"] = _deposit
data['_metadata'] = _metadata
return data, pid, recid
def create_object(bucket, record_dict):
"""Object creation inside the bucket using the file and its content."""
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
files_meta, num_of_iiif_valid_files = generate_files_metadata(
bucket, record_dict['_files'])
# If there are any iiif valid image files, iiif manifest api is added on
# record metadata.
iiif_manifest_url = ''
if num_of_iiif_valid_files > 0:
iiif_manifest_url = '/record/{0}/iiif/manifest.json'.format(
provider.pid.pid_value)
deposit_dict = record_dict['_deposit']
deposit_dict['iiif_manifest'] = iiif_manifest_url
data = {
'pid_value': provider.pid.pid_value,
'_deposit': deposit_dict,
'_files': files_meta,
}
# from invenio_records_files.api import Record as RecordFile
record = RecordFile.create(data, id_=rec_uuid)
# connect to record and bucket
db.session.add(RecordsBuckets(
record_id=record.id,
bucket_id=bucket.id,
))
db.session.commit()
def zenodo_record_minter(record_uuid, data):
"""Mint record identifiers."""
assert 'recid' not in data
provider = RecordIdProvider.create(
object_type='rec', object_uuid=record_uuid)
data['recid'] = int(provider.pid.pid_value)
return provider.pid
def cap_record_minter(record_uuid, data):
"""Mint record identifiers."""
assert 'recid' not in data
provider = RecordIdProvider.create(object_type='rec',
object_uuid=record_uuid)
data['recid'] = int(provider.pid.pid_value)
return provider.pid
def construct_record(collection, metadata, creator_id, schema):
"""Create a record."""
# Creating a uuid4
recid = uuid4()
# Creating a PID for the record
provider = RecordIdProvider.create(object_type='rec', object_uuid=recid)
pid = provider.pid.pid_value
data = {}
_deposit = {}
_metadata = metadata
_deposit['created_by'] = creator_id
# _deposit['id'] = recid.int
_deposit['pid'] = {"revision_id": 0, "type": "recid", "value": pid}
_deposit['status'] = "draft"
_deposit['pid_value'] = pid
_deposit['control_number'] = pid
if collection.parent and collection.parent.parent.name == 'CERNAnalysisPreservation':
data['experiment'] = collection.parent.name
data['collections'] = [collection.name]
data['document_type_human'] = get_document_type(collection.name)
data['$schema'] = schema
# data["recid"] = recid.int
data["_deposit"] = _deposit
data['_metadata'] = _metadata
data['_files'] = []
return data, pid, recid
def record_minter(record_uuid, data):
parent = data.get('conceptrecid')
if not parent:
parent_pid = RecordIdProvider.create(object_type='rec',
object_uuid=None,
status=PIDStatus.REGISTERED).pid
data['conceptrecid'] = parent_pid.pid_value
else:
parent_pid = PersistentIdentifier.get(
pid_type=RecordIdProvider.pid_type, pid_value=parent)
provider = RecordIdProvider.create('rec', record_uuid)
data['recid'] = provider.pid.pid_value
versioning = PIDVersioning(parent=parent_pid)
versioning.insert_child(child=provider.pid)
return provider.pid
def get_pid():
"""Generates a new PID for a record."""
record_uuid = uuid.uuid4()
provider = RecordIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
return record_uuid, provider.pid.pid_value
def document_minter(object_uuid, data):
provider = RecordIdProvider.create(
object_type='rec',
object_uuid=object_uuid,
)
data['pid'] = provider.pid.pid_value
return provider.pid
def author_minter(record_uuid, data):
"""Minter for authors."""
pid_field = current_app.config['PIDSTORE_RECID_FIELD']
assert pid_field not in data
provider = RecordIdProvider.create(object_type='rec',
object_uuid=record_uuid,
pid_type='authid')
data[pid_field] = provider.pid.pid_value
return provider.pid
def record_pid_minter(record_uuid, data):
"""Mint loan identifiers."""
assert "pid" not in data
provider = RecordIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
data["pid"] = provider.pid.pid_value
return provider.pid
def record(db):
"""Record fixture."""
rec_uuid = uuid.uuid4()
provider = RecordIdProvider.create(
object_type='rec', object_uuid=rec_uuid)
record = Record.create({
'control_number': provider.pid.pid_value,
'title': 'TestDefault',
}, id_=rec_uuid)
db.session.commit()
return record
def recid_minter(record_uuid, data):
"""RERIOLS recid minter."""
assert 'recid' not in data
provider = RecordIdProvider.create(
object_type='rec',
object_uuid=record_uuid
)
pid = provider.pid
data['recid'] = pid.pid_value
return pid
def files():
"""Load files."""
data_path = os.path.join(os.path.dirname(__file__), 'data')
# Create location
loc = Location(name='local', uri=data_path, default=True)
db.session.commit()
# Bucket
bucket = Bucket.create(loc)
# Example files from the data folder
example_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'jpgfile.jpg',
'pngfile.png',
)
# Create single file records
for f in example_files:
with open(os.path.join(data_path, f), 'rb') as fp:
create_object(bucket, f, fp)
# Create a multi-file record
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
'files': []
}
# Template to create different files
template_file = {
'uri': '/files/{0}/{1}',
'key': '',
'bucket': str(bucket.id),
'local': True
}
for filename in example_files:
file_data = template_file.copy()
file_data['uri'] = file_data['uri'].format(str(bucket.id), filename)
file_data['key'] = filename
data['files'].append(file_data)
Record.create(data, id_=rec_uuid)
db.session.commit()
def test_index_siblings(app, db, version_pids):
"""Test the index_siblings method."""
# Create a pid relation with 3 children
data_v1 = {'body': u'test_body',
'title': u'test_title'}
data_v2 = {'body': u'test_body2',
'title': u'test_title2'}
data_v3 = {'body': u'test_body3',
'title': u'test_title3'}
rec_v1 = Record.create(data_v1)
parent_pid = RecordIdProvider.create(object_type='rec',
object_uuid=None,
status=PIDStatus.REGISTERED).pid
data_v1['conceptrecid'] = parent_pid.pid_value
provider = RecordIdProvider.create('rec', rec_v1.id)
data_v1['recid'] = provider.pid.pid_value
versioning = PIDNodeVersioning(pid=parent_pid)
versioning.insert_child(child_pid=provider.pid)
rec_v2 = Record.create(data_v2)
data_v2['conceptrecid'] = parent_pid.pid_value
provider_v2 = RecordIdProvider.create('rec', rec_v2.id)
data_v2['recid'] = provider_v2.pid.pid_value
versioning.insert_child(child_pid=provider_v2.pid)
rec_v3 = Record.create(data_v3)
data_v3['conceptrecid'] = parent_pid.pid_value
provider_v3 = RecordIdProvider.create('rec', rec_v3.id)
data_v3['recid'] = provider.pid.pid_value
versioning.insert_child(child_pid=provider_v3.pid)
db.session.commit()
with patch('invenio_indexer.api.RecordIndexer.index_by_id') as mock:
index_siblings(provider.pid, include_pid=True, eager=True,
with_deposits=False)
mock.assert_any_call(str(provider.pid.object_uuid))
mock.assert_any_call(str(provider_v2.pid.object_uuid))
mock.assert_any_call(str(provider_v3.pid.object_uuid))
def record_minter(object_uuid, data):
# pid = PersistentIdentifier.create(
# pid_type='recid',
# pid_value="xxxx",
# object_uuid=object_uuid,
# )
provider = RecordIdProvider.create(
object_type='rec',
object_uuid=object_uuid,
)
data['id'] = provider.pid.pid_value
return provider.pid, data
def files():
"""Load files."""
data_path = os.path.join(os.path.dirname(__file__), 'data')
# Create location
loc = Location(name='local', uri=data_path, default=True)
db.session.commit()
# Bucket
bucket = Bucket.create(loc)
# Example files from the data folder
example_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'jpgfile.jpg',
'pngfile.png',
)
# Create single file records
for f in example_files:
with open(os.path.join(data_path, f), 'rb') as fp:
create_object(bucket, f, fp)
# Create a multi-file record
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {'pid_value': provider.pid.pid_value, 'files': []}
# Template to create different files
template_file = {
'uri': '/files/{0}/{1}',
'key': '',
'bucket': str(bucket.id),
'local': True
}
for filename in example_files:
file_data = template_file.copy()
file_data['uri'] = file_data['uri'].format(str(bucket.id), filename)
file_data['key'] = filename
data['files'].append(file_data)
Record.create(data, id_=rec_uuid)
db.session.commit()
def zenodo_record_minter(record_uuid, data):
"""Mint record identifier (and DOI)."""
if 'recid' in data:
recid = PersistentIdentifier.get('recid', data['recid'])
recid.assign('rec', record_uuid)
recid.register()
else:
recid = RecordIdProvider.create(
object_type='rec', object_uuid=record_uuid).pid
data['recid'] = int(recid.pid_value)
zenodo_doi_minter(record_uuid, data)
return recid
def zenodo_record_minter(record_uuid, data):
"""Mint record identifier (and DOI)."""
if 'recid' in data:
recid = PersistentIdentifier.get('recid', data['recid'])
recid.assign('rec', record_uuid)
recid.register()
else:
recid = RecordIdProvider.create(
object_type='rec', object_uuid=record_uuid).pid
data['recid'] = int(recid.pid_value)
zenodo_doi_minter(record_uuid, data)
zenodo_oaiid_minter(record_uuid, data)
return recid
def create_object(bucket, file_name, stream):
"""Object creation inside the bucket using the file and its content."""
obj = ObjectVersion.create(bucket, file_name, stream=stream)
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value':
provider.pid.pid_value,
'files': [{
'uri': '/files/{0}/{1}'.format(str(bucket.id), file_name),
'key': file_name,
'size': obj.file.size,
'bucket': str(bucket.id),
'local': True
}]
}
Record.create(data, id_=rec_uuid)
def create_object(bucket, file_name, stream):
"""Object creation inside the bucket using the file and its content."""
obj = ObjectVersion.create(bucket, file_name, stream=stream)
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
'files': [
{
'uri': '/files/{0}/{1}'.format(str(bucket.id), file_name),
'key': file_name,
'size': obj.file.size,
'bucket': str(bucket.id),
'local': True
}
]
}
Record.create(data, id_=rec_uuid)
def fixtures():
"""Command for working with test data."""
temp_path = os.path.join(os.path.dirname(__file__), 'instance/temp')
demo_files_path = os.path.join(os.path.dirname(__file__), 'demo_files')
# Create location
loc = Location(name='local', uri=temp_path, default=True)
db.session.add(loc)
db.session.commit()
# Example files from the data folder
demo_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'pdffile.pdf',
'jpgfile.jpg',
'pngfile.png',
'pdffile.pdf',
)
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
}
record = Record.create(data, id_=rec_uuid)
bucket = Bucket.create()
RecordsBuckets.create(record=record.model, bucket=bucket)
# Add files to the record
for f in demo_files:
with open(os.path.join(demo_files_path, f), 'rb') as fp:
record.files[f] = fp
record.files.flush()
record.commit()
db.session.commit()
def zenodo_record_minter(record_uuid, data):
"""Zenodo record minter.
Mint, or register if previously minted, the Concept RECID and RECID.
Mint the Concept DOI and DOI.
"""
if 'conceptrecid' not in data:
zenodo_concept_recid_minter(record_uuid, data)
if 'recid' in data:
recid = PersistentIdentifier.get('recid', data['recid'])
recid.assign('rec', record_uuid)
recid.register()
else:
recid = RecordIdProvider.create(
object_type='rec', object_uuid=record_uuid).pid
data['recid'] = int(recid.pid_value)
zenodo_doi_minter(record_uuid, data)
oaiid_minter(record_uuid, data)
if 'conceptdoi' not in data:
zenodo_concept_doi_minter(record_uuid, data)
return recid
def zenodo_record_minter(record_uuid, data):
"""Zenodo record minter.
Mint, or register if previously minted, the Concept RECID and RECID.
Mint the Concept DOI and DOI.
"""
if 'conceptrecid' not in data:
zenodo_concept_recid_minter(record_uuid, data)
if 'recid' in data:
recid = PersistentIdentifier.get('recid', data['recid'])
recid.assign('rec', record_uuid)
recid.register()
else:
recid = RecordIdProvider.create(object_type='rec',
object_uuid=record_uuid).pid
data['recid'] = int(recid.pid_value)
zenodo_doi_minter(record_uuid, data)
oaiid_minter(record_uuid, data)
if 'conceptdoi' not in data:
zenodo_concept_doi_minter(record_uuid, data)
return recid
def record_minter(record_uuid, data):
provider = RecordIdProvider.create('rec', record_uuid)
data['recid'] = provider.pid.pid_value
return provider.pid
