def delete(self):
"""Delete a deposit."""
deposit_pid = self.pid
pid_value = deposit_pid.pid_value
record_pid = RecordUUIDProvider.get(pid_value).pid
version_master = PIDVersioning(child=record_pid)
# every deposit has a parent version after the 2.1.0 upgrade
# except deleted ones. We check the parent version in case of a delete
# revert.
assert version_master is not None, 'Unexpected deposit without versioning.'
# if the record is unpublished hard delete it
if record_pid.status == PIDStatus.RESERVED:
version_master.remove_draft_child()
db.session.delete(record_pid)
# if the parent doesn't have any published records hard delete it
if version_master.parent.status == PIDStatus.RESERVED:
db.session.delete(version_master.parent)
deposit_pid.delete()
# delete all buckets linked to the deposit
res = Bucket.query.join(RecordsBuckets).\
filter(RecordsBuckets.bucket_id == Bucket.id,
RecordsBuckets.record_id == self.id).all()
# remove the deposit from ES
self.indexer.delete(self)
# we call the super of Invenio deposit instead of B2Share deposit as
# Invenio deposit doesn't support the deletion of published deposits
super(InvenioDeposit, self).delete(force=True)
for bucket in res:
bucket.locked = False
bucket.remove()
def doi_ish_view_method(parent_pid_value=0, version=0):
"""DOI-like item version endpoint view.
:param pid: PID value.
:returns: Redirect to correct version.
"""
try:
p_pid = PersistentIdentifier.get('parent',
'parent:' + str(parent_pid_value))
except PIDDoesNotExistError:
p_pid = None
if p_pid:
pid_ver = PIDVersioning(parent=p_pid)
all_versions = list(pid_ver.get_children(ordered=True,
pid_status=None))
if version == 0 or version == len(all_versions):
return redirect(
url_for('invenio_records_ui.recid',
pid_value=pid_ver.last_child.pid_value))
elif version <= len(all_versions):
version_pid = all_versions[(version - 1)]
current_app.logger.info(version_pid.__dict__)
if version_pid.status == PIDStatus.REGISTERED:
return redirect(
url_for('invenio_records_ui.recid',
pid_value=version_pid.pid_value))
return abort(404)
def versioning_published_record(uuid):
"""Migrate a published record."""
record = ZenodoRecord.get_record(uuid)
if 'conceptrecid' in record:
return
# ASSERT ZENODO DOI ONLY!
assert 'conceptrecid' not in record, "Record already migrated"
# doi = PersistentIdentifier.get('doi', str(record['doi']))
# assert is_local_doi(doi.pid_value), 'DOI is not controlled by Zenodo.'
conceptrecid = zenodo_concept_recid_minter(uuid, record)
conceptrecid.register()
recid = PersistentIdentifier.get('recid', str(record['recid']))
pv = PIDVersioning(parent=conceptrecid)
pv.insert_child(recid)
record.commit()
# Some old records have no deposit ID, some don't have '_deposit'
if ('_deposit' in record and 'id' in record['_deposit']
and record['_deposit']['id']):
try:
depid = PersistentIdentifier.get('depid',
str(record['_deposit']['id']))
deposit = ZenodoDeposit.get_record(depid.object_uuid)
deposit['conceptrecid'] = conceptrecid.pid_value
if deposit['_deposit']['status'] == 'draft':
deposit['_deposit']['pid']['revision_id'] = \
deposit['_deposit']['pid']['revision_id'] + 1
deposit.commit()
except PIDDoesNotExistError:
pass
db.session.commit()
def delete(self, delete_published=False, *args, **kwargs):
"""Delete the deposit.
:param delete_published: If True, even deposit of a published record
will be deleted (usually used by admin operations).
:type delete_published: bool
"""
is_published = self['_deposit'].get('pid')
if is_published and not delete_published:
raise PIDInvalidAction()
# Delete the recid
recid = PersistentIdentifier.get(pid_type='recid',
pid_value=self['recid'])
versioning = PIDVersioning(child=recid)
if versioning.exists:
if versioning.draft_child and \
self.pid == versioning.draft_child_deposit:
versioning.remove_draft_child()
if versioning.last_child:
index_siblings(versioning.last_child,
children=versioning.children.all(),
include_pid=True,
neighbors_eager=True,
with_deposits=True)
if recid.status == PIDStatus.RESERVED:
db.session.delete(recid)
if 'conceptrecid' in self:
concept_recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['conceptrecid'])
if concept_recid.status == PIDStatus.RESERVED:
db.session.delete(concept_recid)
# Completely remove bucket
bucket = self.files.bucket
with db.session.begin_nested():
# Remove Record-Bucket link
RecordsBuckets.query.filter_by(record_id=self.id).delete()
mp_q = MultipartObject.query_by_bucket(bucket)
# Remove multipart objects
Part.query.filter(
Part.upload_id.in_(
mp_q.with_entities(
MultipartObject.upload_id).subquery())).delete(
synchronize_session='fetch')
mp_q.delete(synchronize_session='fetch')
bucket.locked = False
bucket.remove()
depid = kwargs.get('pid', self.pid)
if depid:
depid.delete()
# NOTE: We call the parent of Deposit, invenio_records.api.Record since
# we need to completely override eveything that the Deposit.delete
# method does.
return super(Deposit, self).delete(*args, **kwargs)
def _publish_new(self, id_=None):
"""Publish new deposit with communities handling."""
dep_comms = set(self.pop('communities', []))
record = super(ZenodoDeposit, self)._publish_new(id_=id_)
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
if pv.children.count() > 1:
files_set = set(f.get_version().file.checksum for f in self.files)
for prev_recid in pv.children.all()[:-1]:
rec = ZenodoRecord.get_record(prev_recid.object_uuid)
prev_files_set = set(f.get_version().file.checksum for f in
rec.files)
if files_set == prev_files_set:
raise VersioningFilesError()
prev_recid = pv.children.all()[-2]
rec_comms = set(ZenodoRecord.get_record(
prev_recid.get_assigned_object()).get('communities', []))
else:
rec_comms = set()
record = self._sync_communities(dep_comms, rec_comms, record)
record.commit()
# Update the concept recid redirection
pv.update_redirect()
RecordDraft.unlink(record.pid, self.pid)
index_siblings(record.pid, neighbors_eager=True, with_deposits=True)
return record
def delete(self):
"""Delete a deposit."""
deposit_pid = self.pid
pid_value = deposit_pid.pid_value
record_pid = RecordUUIDProvider.get(pid_value).pid
version_master = PIDVersioning(child=record_pid)
# every deposit has a parent version after the 2.1.0 upgrade
# except deleted ones. We check the parent version in case of a delete
# revert.
assert version_master is not None, 'Unexpected deposit without versioning.'
# if the record is unpublished hard delete it
if record_pid.status == PIDStatus.RESERVED:
version_master.remove_draft_child()
db.session.delete(record_pid)
# if the parent doesn't have any published records hard delete it
if version_master.parent.status == PIDStatus.RESERVED:
db.session.delete(version_master.parent)
deposit_pid.delete()
# delete all buckets linked to the deposit
res = Bucket.query.join(RecordsBuckets).\
filter(RecordsBuckets.bucket_id == Bucket.id,
RecordsBuckets.record_id == self.id).all()
# remove the deposit from ES
self.indexer.delete(self)
# we call the super of Invenio deposit instead of B2Share deposit as
# Invenio deposit doesn't support the deletion of published deposits
super(InvenioDeposit, self).delete(force=True)
for bucket in res:
bucket.locked = False
bucket.remove()
def _publish_new(self, id_=None):
"""Publish new deposit with communities handling."""
dep_comms = set(self.pop('communities', []))
record = super(ZenodoDeposit, self)._publish_new(id_=id_)
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
if pv.children.count() > 1:
files_set = set(f.get_version().file.checksum for f in self.files)
for prev_recid in pv.children.all()[:-1]:
rec = ZenodoRecord.get_record(prev_recid.object_uuid)
prev_files_set = set(f.get_version().file.checksum for f in
rec.files)
if files_set == prev_files_set:
raise VersioningFilesError()
prev_recid = pv.children.all()[-2]
rec_comms = set(ZenodoRecord.get_record(
prev_recid.get_assigned_object()).get('communities', []))
else:
rec_comms = set()
record = self._sync_communities(dep_comms, rec_comms, record)
record.commit()
# Update the concept recid redirection
pv.update_redirect()
RecordDraft.unlink(record.pid, self.pid)
index_siblings(record.pid, neighbors_eager=True, with_deposits=True)
return record
def create_deposit_and_record(pid_value, owner):
"""Utility function for creating records and deposits."""
recid = PersistentIdentifier.create(
'recid', pid_value, status=PIDStatus.RESERVED)
pv = PIDVersioning(parent=conceptrecid)
pv.insert_draft_child(recid)
depid = PersistentIdentifier.create(
'depid', pid_value, status=PIDStatus.REGISTERED)
deposit = ZenodoRecord.create({'_deposit': {'id': depid.pid_value},
'conceptrecid': conceptrecid.pid_value,
'recid': recid.pid_value})
deposit.commit()
depid.assign('rec', deposit.id)
record_metadata = deepcopy(minimal_record)
record_metadata['_deposit'] = {'id': depid.pid_value}
record_metadata['conceptrecid'] = conceptrecid.pid_value
record_metadata['recid'] = int(recid.pid_value)
record_metadata['owners'] = [owner.id]
record = ZenodoRecord.create(record_metadata)
zenodo_record_minter(record.id, record)
record.commit()
return (depid, deposit, recid, record)
def preprocess_related_identifiers(pid, record, result):
"""Preprocess related identifiers for record serialization.
Resolves the passed pid to the proper `recid` in order to add related
identifiers from PID relations.
"""
recid_value = record.get('recid')
if pid.pid_type == 'doi' and pid.pid_value == record.get('conceptdoi'):
recid_value = record.get('conceptrecid')
result['metadata']['doi'] = record.get('conceptdoi')
recid = (pid if pid.pid_value == recid_value else PersistentIdentifier.get(
pid_type='recid', pid_value=recid_value))
if recid.pid_value == record.get('conceptrecid'):
pv = PIDVersioning(parent=recid)
else:
pv = PIDVersioning(child=recid)
# Serialize PID versioning as related identifiers
if pv.exists:
rels = serialize_related_identifiers(recid)
if rels:
result['metadata'].setdefault('related_identifiers',
[]).extend(rels)
return result
def test_version_pids_create(app, db):
# Create a child, initialize the Versioning API and create a parent
assert PersistentIdentifier.query.count() == 0
# Create a child
h1v1 = PersistentIdentifier.create('recid', '12345', object_type='rec',
status=PIDStatus.REGISTERED)
assert PersistentIdentifier.query.count() == 1
pv = PIDVersioning(child=h1v1)
# Create a parent
pv.create_parent('12345.parent')
assert PersistentIdentifier.query.count() == 2
assert pv.parent.get_redirect() == h1v1
assert pv.parent.status == PIDStatus.REDIRECTED
# Make sure 'pid_type', 'object_type' and 'status' are inherited from child
assert pv.parent.pid_type == pv.child.pid_type
assert pv.parent.object_type == pv.child.object_type
pr = PIDRelation.query.one()
assert pr.child == h1v1
assert pr.parent == pv.parent
VERSION = resolve_relation_type_config('version').id
assert pr.relation_type == VERSION
assert pr.index == 0
def test_deposit_versioning_draft_child_unlinking_bug(app, db, communities,
deposit, deposit_file):
"""
Bug with draft_child_deposit unlinking.
Bug where a draft_child_deposit was unlinked from a new version draft,
when another version of a record was edited and published.
"""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
# Initiate a new version draft
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
assert pv.draft_child_deposit
assert pv.draft_child
deposit_v1.edit()
deposit_v1 = deposit_v1.edit()
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
pv = PIDVersioning(child=recid_v1)
# Make sure the draft child deposit was not unliked due to publishing of
# the edited draft
assert pv.draft_child_deposit
assert pv.draft_child
def delete(self, delete_published=False, *args, **kwargs):
"""Delete the deposit.
:param delete_published: If True, even deposit of a published record
will be deleted (usually used by admin operations).
:type delete_published: bool
"""
is_published = self['_deposit'].get('pid')
if is_published and not delete_published:
raise PIDInvalidAction()
# Delete the recid
recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['recid'])
versioning = PIDVersioning(child=recid)
if versioning.exists:
if versioning.draft_child and \
self.pid == versioning.draft_child_deposit:
versioning.remove_draft_child()
if versioning.last_child:
index_siblings(versioning.last_child,
children=versioning.children.all(),
include_pid=True,
neighbors_eager=True,
with_deposits=True)
if recid.status == PIDStatus.RESERVED:
db.session.delete(recid)
if 'conceptrecid' in self:
concept_recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['conceptrecid'])
if concept_recid.status == PIDStatus.RESERVED:
db.session.delete(concept_recid)
# Completely remove bucket
bucket = self.files.bucket
with db.session.begin_nested():
# Remove Record-Bucket link
RecordsBuckets.query.filter_by(record_id=self.id).delete()
mp_q = MultipartObject.query_by_bucket(bucket)
# Remove multipart objects
Part.query.filter(
Part.upload_id.in_(mp_q.with_entities(
MultipartObject.upload_id).subquery())
).delete(synchronize_session='fetch')
mp_q.delete(synchronize_session='fetch')
bucket.locked = False
bucket.remove()
depid = kwargs.get('pid', self.pid)
if depid:
depid.delete()
# NOTE: We call the parent of Deposit, invenio_records.api.Record since
# we need to completely override eveything that the Deposit.delete
# method does.
return super(Deposit, self).delete(*args, **kwargs)
def cancel_action(activity_id='0', action_id=0):
"""Next action."""
post_json = request.get_json()
work_activity = WorkActivity()
activity = dict(activity_id=activity_id,
action_id=action_id,
action_version=post_json.get('action_version'),
action_status=ActionStatusPolicy.ACTION_CANCELED,
commond=post_json.get('commond'))
# clear deposit
activity_detail = work_activity.get_activity_detail(activity_id)
if activity_detail is not None:
cancel_item_id = activity_detail.item_id
if cancel_item_id is None:
pid_value = post_json.get('pid_value')
if pid_value is not None:
pid = PersistentIdentifier.get('recid', pid_value)
cancel_item_id = pid.object_uuid
if cancel_item_id is not None:
cancel_record = WekoDeposit.get_record(cancel_item_id)
if cancel_record is not None:
cancel_deposit = WekoDeposit(cancel_record,
cancel_record.model)
cancel_deposit.clear()
# Remove draft child
cancel_pid = PersistentIdentifier.get_by_object(
pid_type='recid',
object_type='rec',
object_uuid=cancel_item_id)
cancel_pv = PIDVersioning(child=cancel_pid)
if cancel_pv.exists:
previous_pid = cancel_pv.previous
if previous_pid is not None:
activity.update(dict(item_id=previous_pid.object_uuid))
cancel_pv.remove_child(cancel_pid)
work_activity.upt_activity_action_status(
activity_id=activity_id,
action_id=action_id,
action_status=ActionStatusPolicy.ACTION_CANCELED)
rtn = work_activity.quit_activity(activity)
if rtn is None:
work_activity.upt_activity_action_status(
activity_id=activity_id,
action_id=action_id,
action_status=ActionStatusPolicy.ACTION_DOING)
return jsonify(code=-1, msg=_('Error! Cannot process quit activity!'))
return jsonify(code=0,
msg=_('success'),
data={
'redirect':
url_for('weko_workflow.display_activity',
activity_id=activity_id)
})
def retrieve_version_master(child_pid):
"""Retrieve the PIDVersioning from a child PID."""
if type(child_pid).__name__ == "FetchedPID":
# when getting a pid-like object from elasticsearch
child_pid = child_pid.provider.get(child_pid.pid_value).pid
parent_pid = PIDVersioning(child=child_pid).parent
if not parent_pid:
return None
return PIDVersioning(parent=parent_pid)
def serialize_related_identifiers(pid):
"""Serialize PID Versioning relations as related_identifiers metadata."""
pv = PIDVersioning(child=pid)
related_identifiers = []
if pv.exists:
rec = ZenodoRecord.get_record(pid.get_assigned_object())
# External DOI records don't have Concept DOI
if 'conceptdoi' in rec:
ri = {
'scheme': 'doi',
'relation': 'isVersionOf',
'identifier': rec['conceptdoi']
}
related_identifiers.append(ri)
# TODO: We do not serialize previous/next versions to
# related identifiers because of the semantic-versioning cases
# (e.g. GitHub releases of minor versions)
#
# children = pv.children.all()
# idx = children.index(pid)
# left = children[:idx]
# right = children[idx + 1:]
# for p in left:
# rec = ZenodoRecord.get_record(p.get_assigned_object())
# ri = {
# 'scheme': 'doi',
# 'relation': 'isNewVersionOf',
# 'identifier': rec['doi']
# }
# related_identifiers.append(ri)
# for p in right:
# rec = ZenodoRecord.get_record(p.get_assigned_object())
# ri = {
# 'scheme': 'doi',
# 'relation': 'isPreviousVersionOf',
# 'identifier': rec['doi']
# }
# related_identifiers.append(ri)
pv = PIDVersioning(parent=pid)
if pv.exists:
for p in pv.children:
rec = ZenodoRecord.get_record(p.get_assigned_object())
ri = {
'scheme': 'doi',
'relation': 'hasVersion',
'identifier': rec['doi']
}
related_identifiers.append(ri)
return related_identifiers
def create_versioned_record(recid_value, conceptrecid):
"""Utility function for creating versioned records."""
recid = PersistentIdentifier.create(
'recid', recid_value, status=PIDStatus.RESERVED)
pv = PIDVersioning(parent=conceptrecid)
pv.insert_draft_child(recid)
record_metadata = deepcopy(minimal_record)
record_metadata['conceptrecid'] = conceptrecid.pid_value
record_metadata['recid'] = int(recid.pid_value)
record = ZenodoRecord.create(record_metadata)
zenodo_record_minter(record.id, record)
record.commit()
return recid, record
def versioning_new_deposit(uuid):
"""Migrate a yet-unpublished deposit to a versioning scheme."""
deposit = ZenodoDeposit.get_record(uuid)
if 'conceptrecid' in deposit:
return
# ASSERT ZENODO DOI ONLY!
assert 'conceptrecid' not in deposit, 'Concept RECID already in record.'
conceptrecid = zenodo_concept_recid_minter(uuid, deposit)
recid = PersistentIdentifier.get('recid', str(deposit['recid']))
depid = PersistentIdentifier.get('depid', str(deposit['_deposit']['id']))
pv = PIDVersioning(parent=conceptrecid)
pv.insert_draft_child(recid)
RecordDraft.link(recid, depid)
deposit.commit()
db.session.commit()
def test_propagation_with_newversion_open(
app, db, users, communities, deposit, deposit_file):
"""Adding old versions to a community should propagate to all drafts."""
# deposit['communities'] = ['c1', 'c2']
deposit_v1 = publish_and_expunge(db, deposit)
deposit_v1 = deposit_v1.edit()
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
# New version in 'deposit_v2' has not been published yet
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
# depid_v1_value = deposit_v1['_deposit']['id']
# depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1['communities'] = ['c1', 'c2', ]
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
c1_api = ZenodoCommunity('c1')
c1_api.accept_record(record_v1, pid=recid_v1)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
assert deposit_v2['communities'] == ['c1', 'c2']
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
assert record_v2['communities'] == ['c1', ]
def check_records_migration(app):
"""Check that a set of records have been migrated."""
expected_records = _load_json('expected_records.json')
for exp_record in expected_records:
db_record = Record.get_record(exp_record['id'], with_deleted=True)
assert str(db_record.created) == exp_record['created']
# If the record is deleted there is no metadata to check
if db_record.model.json is None:
continue
# Check that the parent pid is minted properly
parent_pid = b2share_parent_pid_fetcher(exp_record['id'], db_record)
fetched_pid = b2share_record_uuid_fetcher(exp_record['id'], db_record)
record_pid = PersistentIdentifier.get(fetched_pid.pid_type,
fetched_pid.pid_value)
assert PIDVersioning(
record_pid).parent.pid_value == parent_pid.pid_value
# Remove the parent pid as it has been added by the migration
db_record['_pid'].remove({
'type': RecordUUIDProvider.parent_pid_type,
'value': parent_pid.pid_value,
})
# The OAI-PMH identifier has been modified by the migration
if db_record.get('_oai'):
oai_prefix = app.config.get('OAISERVER_ID_PREFIX', 'oai:')
record_id = exp_record['metadata']['_deposit']['id']
assert db_record['_oai']['id'] == str(oai_prefix) + record_id
exp_record['metadata']['_oai']['id'] = db_record['_oai']['id']
assert db_record == exp_record['metadata']
def test_autoadd_explicit_newversion(
app, db, users, communities, deposit, deposit_file,
communities_autoadd_enabled):
"""Explicitly the autoadded communities in a new version."""
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit_v1.fetch_published()
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1_value = recid_v1.pid_value
deposit_v1 = deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
depid_v2_value = depid_v2.pid_value
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2['communities'] = ['ecfunded', 'grants_comm', 'zenodo']
deposit_v2['grants'] = [{'title': 'SomeGrant'}, ]
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
depid_v2, deposit_v2 = deposit_resolver.resolve(depid_v2_value)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
assert record_v1.get('communities', []) == ['grants_comm', ]
assert deposit_v1.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
assert record_v2.get('communities', []) == ['grants_comm', ]
assert deposit_v2.get('communities', []) == ['ecfunded', 'grants_comm',
'zenodo']
def update_record_statistics(start_date=None, end_date=None):
"""Update "_stats" field of affected records."""
start_date = dateutil_parse(start_date) if start_date else None
end_date = dateutil_parse(end_date) if start_date else None
aggr_configs = {}
if not start_date and not end_date:
start_date = datetime.utcnow()
end_date = datetime.utcnow()
for aggr_name, aggr_cfg in current_stats.aggregations.items():
aggr = aggr_cfg.cls(name=aggr_cfg.name, **aggr_cfg.params)
if not Index(aggr.index, using=aggr.client).exists():
if not Index(aggr.event_index, using=aggr.client).exists():
start_date = min(start_date, datetime.utcnow())
else:
start_date = min(start_date,
aggr._get_oldest_event_timestamp())
# Retrieve the last two bookmarks
bookmarks = aggr.list_bookmarks(limit=2)
if len(bookmarks) >= 1:
end_date = max(
end_date,
datetime.strptime(bookmarks[0].date, aggr.doc_id_suffix))
if len(bookmarks) == 2:
start_date = min(
start_date,
datetime.strptime(bookmarks[1].date, aggr.doc_id_suffix))
aggr_configs[aggr.index] = aggr
elif start_date and end_date:
for aggr_name, aggr_cfg in current_stats.aggregations.items():
aggr = aggr_cfg.cls(name=aggr_cfg.name, **aggr_cfg.params)
aggr_configs[aggr.index] = aggr
else:
return
# Get conceptrecids for all the affected records between the two dates
conceptrecids = set()
for aggr_index, aggr in aggr_configs.items():
query = Search(
using=aggr.client,
index=aggr.index,
doc_type=aggr.doc_type,
).filter('range',
timestamp={
'gte':
start_date.replace(microsecond=0).isoformat() + '||/d',
'lte':
end_date.replace(microsecond=0).isoformat() + '||/d'
}).source(include='conceptrecid')
conceptrecids |= {b.conceptrecid for b in query.scan()}
indexer = RecordIndexer()
for concpetrecid_val in conceptrecids:
conceptrecid = PersistentIdentifier.get('recid', concpetrecid_val)
pv = PIDVersioning(parent=conceptrecid)
children_recids = pv.children.all()
indexer.bulk_index([str(p.object_uuid) for p in children_recids])
def get_data(self, record_id, query_date=None, get_period=False):
"""Public interface of _get_data."""
result = dict(total=0, country=dict(), period=list())
recid = PersistentIdentifier.query.filter_by(
pid_type='recid', object_uuid=record_id).first()
if recid:
versioning = PIDVersioning(child=recid)
if not versioning.exists:
return self._get_data(record_id, query_date, get_period)
_data = list(
self._get_data(record_id=child.object_uuid, get_period=True)
for child in versioning.children.all())
countries = result['country']
for _idx in _data:
for key, value in _idx['country'].items():
countries[key] = countries.get(key, 0) + value
result['total'] = result['total'] + _idx['total']
result['period'] = _idx.get('period', [])
return result
def indexer_receiver(sender, json=None, record=None, index=None,
**dummy_kwargs):
"""Connect to before_record_index signal to transform record for ES."""
if not index.startswith('records-') or record.get('$schema') is None:
return
# Remove files from index if record is not open access.
if json['access_right'] != 'open' and '_files' in json:
del json['_files']
else:
# Compute file count and total size
files = json.get('_files', [])
json['filecount'] = len(files)
json['size'] = sum([f.get('size', 0) for f in files])
pid = PersistentIdentifier.query.filter(
PersistentIdentifier.object_uuid == record.id,
PersistentIdentifier.pid_type == current_pidrelations.primary_pid_type,
).one_or_none()
if pid:
pv = PIDVersioning(child=pid)
if pv.exists:
relations = serialize_relations(pid)
else:
relations = {'version': [{'is_last': True, 'index': 0}, ]}
if relations:
json['relations'] = relations
rels = serialize_related_identifiers(pid)
if rels:
json.setdefault('related_identifiers', []).extend(rels)
# Remove internal data.
if '_internal' in json:
del json['_internal']
def indexer_receiver(sender, json=None, record=None, index=None,
**dummy_kwargs):
"""Connect to before_record_index signal to transform record for ES.
In order to avoid that a record and published deposit differs (e.g. if an
embargo task updates the record), every time we index a record we also
index the deposit and overwrite the content with that of the record.
:param sender: Sender of the signal.
:param json: JSON to be passed for the elastic search.
:type json: `invenio_records.api.Deposit`
:param record: Indexed deposit record.
:type record: `invenio_records.api.Deposit`
:param index: Elasticsearch index name.
:type index: str
"""
if not index.startswith('deposits-records-'):
return
if not isinstance(record, ZenodoDeposit):
record = ZenodoDeposit(record, model=record.model)
if record['_deposit']['status'] == 'published':
schema = json['$schema']
pub_record = record.fetch_published()[1]
# Temporarily set to draft mode to ensure that `clear` can be called
json['_deposit']['status'] = 'draft'
json.clear()
json.update(copy.deepcopy(pub_record.replace_refs()))
# Set back to published mode and restore schema.
json['_deposit']['status'] = 'published'
json['$schema'] = schema
json['_updated'] = pub_record.updated
else:
json['_updated'] = record.updated
json['_created'] = record.created
# Compute filecount and total file size
files = json.get('_files', [])
json['filecount'] = len(files)
json['size'] = sum([f.get('size', 0) for f in files])
recid = record.get('recid')
if recid:
pid = PersistentIdentifier.get('recid', recid)
pv = PIDVersioning(child=pid)
relations = serialize_relations(pid)
if pv.exists:
if pv.draft_child_deposit:
is_last = (pv.draft_child_deposit.pid_value
== record['_deposit']['id'])
relations['version'][0]['is_last'] = is_last
relations['version'][0]['count'] += 1
else:
relations = {'version': [{'is_last': True, 'index': 0}, ]}
if relations:
json['relations'] = relations
def record_minter(record_uuid, data):
parent = data.get('conceptrecid')
if not parent:
parent_pid = RecordIdProvider.create(object_type='rec',
object_uuid=None,
status=PIDStatus.REGISTERED).pid
data['conceptrecid'] = parent_pid.pid_value
else:
parent_pid = PersistentIdentifier.get(
pid_type=RecordIdProvider.pid_type, pid_value=parent)
provider = RecordIdProvider.create('rec', record_uuid)
data['recid'] = provider.pid.pid_value
versioning = PIDVersioning(parent=parent_pid)
versioning.insert_child(child=provider.pid)
return provider.pid
def get_irs(record, community_id=None, pid=None):
"""Get all inclusion requests for given record and community.
:param record: record for which the inclusion requests are fetched.
This includes all of the record's versions.
:param community_id: Narrow down the query to given community.
Query for all communities if 'None'.
"""
if not pid:
pid = PersistentIdentifier.get('recid', record['recid'])
pv = PIDVersioning(child=pid)
if pv.exists:
sq = pv.children.with_entities(
PersistentIdentifier.object_uuid).subquery()
filter_cond = [
InclusionRequest.id_record.in_(sq),
]
if community_id:
filter_cond.append(
InclusionRequest.id_community == community_id)
q = (db.session.query(InclusionRequest).filter(*filter_cond))
else:
q = InclusionRequest.query.filter_by(id_record=record.id).order_by(
InclusionRequest.id_community)
return q
def create(cls, data, id_=None):
"""Create a deposit.
Adds bucket creation immediately on deposit creation.
"""
bucket = Bucket.create(
quota_size=current_app.config['ZENODO_BUCKET_QUOTA_SIZE'],
max_file_size=current_app.config['ZENODO_MAX_FILE_SIZE'],
)
data['_buckets'] = {'deposit': str(bucket.id)}
deposit = super(ZenodoDeposit, cls).create(data, id_=id_)
RecordsBuckets.create(record=deposit.model, bucket=bucket)
recid = PersistentIdentifier.get(
'recid', str(data['recid']))
conceptrecid = PersistentIdentifier.get(
'recid', str(data['conceptrecid']))
depid = PersistentIdentifier.get(
'depid', str(data['_deposit']['id']))
PIDVersioning(parent=conceptrecid).insert_draft_child(child=recid)
RecordDraft.link(recid, depid)
return deposit
def migrate_concept_recid_sips(recid, overwrite=False):
"""Create Bagit metadata for SIPs."""
pid = PersistentIdentifier.get('recid', recid)
pv = PIDVersioning(parent=pid)
all_sips = []
for child in pv.children:
pid, rec = record_resolver.resolve(child.pid_value)
rsips = RecordSIP.query.filter_by(pid_id=pid.id).order_by(
RecordSIP.created)
all_sips.append([rs.sip.id for rs in rsips])
base_sip_id = None
for sipv in all_sips:
for idx, sip_id in enumerate(sipv):
sip = SIP.query.get(sip_id)
base_sip = SIP.query.get(base_sip_id) if base_sip_id else None
bia = BagItArchiver(SIPApi(sip),
patch_of=base_sip,
include_all_previous=(idx > 0))
bmeta = BagItArchiver.get_bagit_metadata(sip)
if (not bmeta) or overwrite:
bia.save_bagit_metadata(overwrite=True)
base_sip_id = sip_id
db.session.commit()
def get_all_deposit_siblings(deposit):
"""Get all siblings of the deposit."""
from invenio_pidstore.models import PersistentIdentifier
from invenio_pidrelations.contrib.versioning import PIDVersioning
recid = deposit['recid']
rec_pid = PersistentIdentifier.get(pid_type='recid', pid_value=str(recid))
pv = PIDVersioning(child=rec_pid)
return [pid.get_assigned_object() for pid in pv.children]
def test_record_delete_v2(mocker, app, db, users, deposit, deposit_file):
"""Delete a record (only last version) with multiple versions."""
dc_mock = mocker.patch(
'invenio_pidstore.providers.datacite.DataCiteMDSClient')
deposit_v1 = publish_and_expunge(db, deposit)
recid_v1, record_v1 = deposit.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
# Stash a copy of v1 for later
rec1 = deepcopy(record_v1)
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
# Stash a copy of v2 for later
rec2 = deepcopy(record_v2)
rec2_id = str(record_v2.id)
assert dc_mock().metadata_delete.call_count == 0
# Remove the first version
delete_record(rec2_id, 'spam', users[0]['id'])
# Make sure all PIDs are deleted
assert PID.get('doi', rec2['doi']).status == PIDStatus.DELETED
assert PID.get('recid', rec2['recid']).status == PIDStatus.DELETED
assert PID.get('depid', rec2['_deposit']['id']).status == PIDStatus.DELETED
# Concept DOI should be left registered
assert PID.get('doi', rec2['conceptdoi']).status == PIDStatus.REGISTERED
# Make sure conceptrecid is redirecting to v1
crecid = PID.get('recid', rec2['conceptrecid'])
assert crecid.status == PIDStatus.REDIRECTED
assert crecid.get_redirect() == PID.get('recid', rec1['recid'])
# Make sure the v1 PIDs are kept intact
assert PID.get('oai', rec1['_oai']['id']).status == PIDStatus.REGISTERED
assert PID.get('doi', rec1['doi']).status == PIDStatus.REGISTERED
assert PID.get('recid', rec1['recid']).status == PIDStatus.REGISTERED
assert PID.get('depid', rec1['_deposit']['id']).status == \
PIDStatus.REGISTERED
# Only the v1 DOI should be deleted
assert dc_mock().doi_post.call_count == 2
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.2')
assert dc_mock().doi_post.has_any_call('10.5072/zenodo.1')
assert dc_mock().metadata_delete.call_count == 1
dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.3')
record = Record.get_record(rec2_id)
assert record['removed_by'] == users[0]['id']
assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def test_basic_api(app, db, communities, deposit, deposit_file):
"""Test basic workflow using Deposit and Communities API."""
deposit_v1 = publish_and_expunge(db, deposit)
depid_v1_value = deposit_v1['_deposit']['id']
recid_v1, record_v1 = deposit_v1.fetch_published()
recid_v1_value = recid_v1.pid_value
deposit_v1.newversion()
pv = PIDVersioning(child=recid_v1)
depid_v2 = pv.draft_child_deposit
deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object())
deposit_v2.files['file.txt'] = BytesIO(b('file1'))
deposit_v2 = publish_and_expunge(db, deposit_v2)
deposit_v2 = deposit_v2.edit()
# 1. Request for 'c1' and 'c2' through deposit v2
deposit_v2['communities'] = [
'c1',
'c2',
]
deposit_v2 = publish_and_expunge(db, deposit_v2)
recid_v2, record_v2 = deposit_v2.fetch_published()
recid_v2_value = recid_v2.pid_value
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
recid_v1, record_v1 = deposit_v1.fetch_published()
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
c1_api = ZenodoCommunity('c1')
c2_api = ZenodoCommunity('c2')
# Inclusion requests should be visible for both records
assert c1_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c1_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
assert c2_api.get_comm_irs(record_v1, pid=recid_v1).count() == 1
assert c2_api.get_comm_irs(record_v2, pid=recid_v2).count() == 1
# Accept to 'c1' through record_v2 (as originally requested),
# and 'c2' through record_v1 (version)
c1_api.accept_record(record_v2, pid=recid_v2)
c2_api.accept_record(record_v1, pid=recid_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
# Accepting individual record to a community should propagate the changes
# to all versions
assert record_v1['communities'] == record_v2['communities'] == \
['c1', 'c2', ]
# Removing 'c1' from deposit_v1 should remove it from two published records
depid_v1, deposit_v1 = deposit_resolver.resolve(depid_v1_value)
deposit_v1 = deposit_v1.edit()
deposit_v1['communities'] = []
deposit_v1 = publish_and_expunge(db, deposit_v1)
recid_v1, record_v1 = record_resolver.resolve(recid_v1_value)
recid_v2, record_v2 = record_resolver.resolve(recid_v2_value)
assert record_v1.get('communities', []) == []
assert record_v2.get('communities', []) == []
def indexer_receiver(sender,
json=None,
record=None,
index=None,
**dummy_kwargs):
"""Connect to before_record_index signal to transform record for ES."""
if not index.startswith('records-') or record.get('$schema') is None:
return
# Remove files from index if record is not open access.
if json['access_right'] != 'open' and '_files' in json:
del json['_files']
else:
# Compute file count and total size
files = json.get('_files', [])
json['filecount'] = len(files)
json['size'] = sum([f.get('size', 0) for f in files])
pid = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_value == str(record['recid']),
PersistentIdentifier.pid_type == 'recid',
PersistentIdentifier.object_uuid == record.id,
).one_or_none()
if pid:
pv = PIDVersioning(child=pid)
if pv.exists:
relations = serialize_relations(pid)
else:
relations = {
'version': [
{
'is_last': True,
'index': 0
},
]
}
if relations:
json['relations'] = relations
rels = serialize_related_identifiers(pid)
if rels:
json.setdefault('related_identifiers', []).extend(rels)
for loc in json.get('locations', []):
if loc.get('lat') and loc.get('lon'):
loc['point'] = {'lat': loc['lat'], 'lon': loc['lon']}
# Remove internal data.
if '_internal' in json:
del json['_internal']
json['_stats'] = build_record_stats(record['recid'],
record.get('conceptrecid'))
custom_es_fields = build_record_custom_fields(json)
for es_field, es_value in custom_es_fields.items():
json[es_field] = es_value
def versioning_link(recids):
"""Link several records into a versioning scheme.
Support cases with some records being already versioned, as long
as they are all within a single versioning scheme.
For example, given the following records:
- 123, 234, 345 (record with 3 versions)
- 543, 432 (record with 2 versions)
- 111 (single non-versioned record)
- 222 (single, non-versioned record)
The following cases are supported (Good) or not supported (Error):
versioning_link 111 123 234 345 (Good - will add 111 as first version)
versioning_link 111 222 (Good, will create new versioning scheme)
versioning_link 345 123 234 (Good - no new records liked, but will reorder
the records in the versioning list)
versioning_link 123 234 543 (Error - trying to link two versioned records)
versioning_link 123 234 (Error - must specify all children)
"""
int_recids = [int(recid) for recid in recids]
if sorted(int_recids) != int_recids and not click.confirm(
u'Requested RECIDS are not in the order of creation. Continue?'):
click.echo(click.style(u'Record linking aborted.', fg='green'))
return
recids_records = [
record_resolver.resolve(recid_val) for recid_val in recids
]
upgraded = [(recid, rec) for recid, rec in recids_records
if 'conceptdoi' in rec]
if len(upgraded) == 1 and not click.confirm(
u'Recid {0} already migrated. Its Concept recid: {1} will be used as'
u'the base for the Concept DOI in the versioning linking. '
u'Continue?'):
return
elif len(upgraded) > 1:
i_recids = [int(recid) for recid in recids]
child_recids = [
int(recid.pid_value)
for recid in PIDVersioning(child=upgraded[0][0]).children.all()
]
if not all(cr in i_recids for cr in child_recids):
click.echo(u'All children recids ({0}) of the upgraded record need'
u' to be specified. Aborting.'.format(
[recid for recid in child_recids]))
return
i_upgraded = [int(recid.pid_value) for recid, rec in upgraded]
if set(child_recids) != set(i_upgraded):
click.echo(u'Found multiple upgraded records {0}, which do not '
u'belong to a single versioning scheme. Aborting.'
u''.format(i_upgraded,
[recid for recid in child_recids]))
return
versioning_link_records(recids)
def _create_records(base_metadata, total, versions, files):
records = []
cur_recid_val = 1
for _ in range(total):
conceptrecid_val = cur_recid_val
conceptrecid = PersistentIdentifier.create('recid',
str(conceptrecid_val),
status='R')
db.session.commit()
versioning = PIDVersioning(parent=conceptrecid)
for ver_idx in range(versions):
recid_val = conceptrecid_val + ver_idx + 1
data = deepcopy(base_metadata)
data.update({
'conceptrecid': str(conceptrecid_val),
'conceptdoi': '10.1234/{}'.format(recid_val),
'recid': recid_val,
'doi': '10.1234/{}'.format(recid_val),
})
record = ZenodoRecord.create(data)
bucket = Bucket.create()
record['_buckets'] = {'record': str(bucket.id)}
record.commit()
RecordsBuckets.create(bucket=bucket, record=record.model)
recid = PersistentIdentifier.create(pid_type='recid',
pid_value=record['recid'],
object_type='rec',
object_uuid=record.id,
status='R')
versioning.insert_child(recid)
file_objects = []
for f in range(files):
filename = 'Test{0}_v{1}.pdf'.format(f, ver_idx)
record.files[filename] = BytesIO(b'1234567890') # 10 bytes
record.files[filename]['type'] = 'pdf'
file_objects.append(record.files[filename].obj)
record.commit()
db.session.commit()
records.append((recid, record, file_objects))
cur_recid_val += versions + 1
return records
def _create_records(base_metadata, total, versions, files):
records = []
cur_recid_val = 1
for _ in range(total):
conceptrecid_val = cur_recid_val
conceptrecid = PersistentIdentifier.create(
'recid', str(conceptrecid_val), status='R')
db.session.commit()
versioning = PIDVersioning(parent=conceptrecid)
for ver_idx in range(versions):
recid_val = conceptrecid_val + ver_idx + 1
data = deepcopy(base_metadata)
data.update({
'conceptrecid': str(conceptrecid_val),
'conceptdoi': '10.1234/{}'.format(recid_val),
'recid': recid_val,
'doi': '10.1234/{}'.format(recid_val),
})
record = ZenodoRecord.create(data)
bucket = Bucket.create()
record['_buckets'] = {'record': str(bucket.id)}
record.commit()
RecordsBuckets.create(bucket=bucket, record=record.model)
recid = PersistentIdentifier.create(
pid_type='recid', pid_value=record['recid'], object_type='rec',
object_uuid=record.id, status='R')
versioning.insert_child(recid)
file_objects = []
for f in range(files):
filename = 'Test{0}_v{1}.pdf'.format(f, ver_idx)
record.files[filename] = BytesIO(b'1234567890') # 10 bytes
record.files[filename]['type'] = 'pdf'
file_objects.append(record.files[filename].obj)
record.commit()
db.session.commit()
records.append((recid, record, file_objects))
cur_recid_val += versions + 1
return records
def alembic_upgrade_database_data(alembic, verbose):
"""Migrate the database data from v2.0.0 to 2.1.0."""
### Add versioning PIDs ###
# Reserve the record PID and versioning PID for unpublished deposits
# Hack: disable record indexing during record migration
from invenio_indexer.api import RecordIndexer
old_index_fn = RecordIndexer.index
RecordIndexer.index = lambda s, record: None
if verbose:
click.secho('migrating deposits and records...')
with db.session.begin_nested():
# Migrate published records
records_pids = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == RecordUUIDProvider.pid_type,
PersistentIdentifier.status == PIDStatus.REGISTERED,
).all()
for rec_pid in records_pids:
if verbose:
click.secho(' record {}'.format(rec_pid.pid_value))
try:
record = Record.get_record(rec_pid.object_uuid)
except NoResultFound:
# The record is deleted but not the PID. Fix it.
rec_pid.status = PIDStatus.DELETED
continue
# Create parent version PID
parent_pid = RecordUUIDProvider.create().pid
version_master = PIDVersioning(parent=parent_pid)
version_master.insert_draft_child(child=rec_pid)
version_master.update_redirect()
migrate_record_metadata(
Record.get_record(rec_pid.object_uuid),
parent_pid
)
# Migrate deposits
deposit_pids = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == DepositUUIDProvider.pid_type,
PersistentIdentifier.status == PIDStatus.REGISTERED,
).all()
for dep_pid in deposit_pids:
if verbose:
click.secho(' deposit {}'.format(dep_pid.pid_value))
try:
deposit = Deposit.get_record(dep_pid.object_uuid)
if deposit['publication_state'] != \
PublicationStates.published.name:
# The record is not published yet. Reserve the PID.
rec_pid = RecordUUIDProvider.create(
object_type='rec',
pid_value=dep_pid.pid_value,
).pid
# Create parent version PID
parent_pid = RecordUUIDProvider.create().pid
assert parent_pid
version_master = PIDVersioning(parent=parent_pid)
version_master.insert_draft_child(child=rec_pid)
else:
# Retrieve previously created version PID
rec_pid = RecordUUIDProvider.get(dep_pid.pid_value).pid
version_master = PIDVersioning(child=rec_pid)
parent_pid = version_master.parent
if not parent_pid:
click.secho(' record {} was deleted, but the deposit has not been removed'.format(rec_pid.pid_value), fg='red')
if parent_pid:
migrate_record_metadata(
Deposit.get_record(dep_pid.object_uuid),
parent_pid
)
except NoResultFound:
# The deposit is deleted but not the PID. Fix it.
dep_pid.status = PIDStatus.DELETED
if verbose:
click.secho('done migrating deposits.')
RecordIndexer.index = old_index_fn
def publish(self):
"""Publish GitHub release as record."""
id_ = uuid.uuid4()
deposit_metadata = dict(self.metadata)
deposit = None
try:
db.session.begin_nested()
# TODO: Add filter on Published releases
previous_releases = self.model.repository.releases.filter_by(
status=ReleaseStatus.PUBLISHED)
versioning = None
stashed_draft_child = None
if previous_releases.count():
last_release = previous_releases.order_by(
Release.created.desc()).first()
last_recid = PersistentIdentifier.get(
'recid', last_release.record['recid'])
versioning = PIDVersioning(child=last_recid)
last_record = ZenodoRecord.get_record(
versioning.last_child.object_uuid)
deposit_metadata['conceptrecid'] = last_record['conceptrecid']
if 'conceptdoi' not in last_record:
last_depid = PersistentIdentifier.get(
'depid', last_record['_deposit']['id'])
last_deposit = ZenodoDeposit.get_record(
last_depid.object_uuid)
last_deposit = last_deposit.registerconceptdoi()
last_recid, last_record = last_deposit.fetch_published()
deposit_metadata['conceptdoi'] = last_record['conceptdoi']
if versioning.draft_child:
stashed_draft_child = versioning.draft_child
versioning.remove_draft_child()
deposit = self.deposit_class.create(deposit_metadata, id_=id_)
deposit['_deposit']['created_by'] = self.event.user_id
deposit['_deposit']['owners'] = [self.event.user_id]
# Fetch the deposit files
for key, url in self.files:
# Make a HEAD request to get GitHub to compute the
# Content-Length.
res = self.gh.api.session.head(url, allow_redirects=True)
# Now, download the file
res = self.gh.api.session.get(url, stream=True,
allow_redirects=True)
if res.status_code != 200:
raise Exception(
"Could not retrieve archive from GitHub: {url}"
.format(url=url)
)
size = int(res.headers.get('Content-Length', 0))
ObjectVersion.create(
bucket=deposit.files.bucket,
key=key,
stream=res.raw,
size=size or None,
mimetype=res.headers.get('Content-Type'),
)
# GitHub-specific SIP store agent
sip_agent = {
'$schema': current_jsonschemas.path_to_url(
current_app.config['SIPSTORE_GITHUB_AGENT_JSONSCHEMA']),
'user_id': self.event.user_id,
'github_id': self.release['author']['id'],
'email': self.gh.account.user.email,
}
deposit.publish(user_id=self.event.user_id, sip_agent=sip_agent)
recid_pid, record = deposit.fetch_published()
self.model.recordmetadata = record.model
if versioning and stashed_draft_child:
versioning.insert_draft_child(stashed_draft_child)
record_id = str(record.id)
db.session.commit()
# Send Datacite DOI registration task
datacite_register.delay(recid_pid.pid_value, record_id)
# Index the record
RecordIndexer().index_by_id(record_id)
except Exception:
db.session.rollback()
# Remove deposit from index since it was not commited.
if deposit and deposit.id:
try:
RecordIndexer().delete(deposit)
except Exception:
current_app.logger.exception(
"Failed to remove uncommited deposit from index.")
raise
def create(cls, data, id_=None, version_of=None):
"""Create a deposit with the optional id.
:params version_of: PID of an existing record. If set, the new record
will be marked as a new version of this referenced record. If no data
is provided the new record will be a copy of this record. Note: this
PID must reference the current last version of a record.
"""
# check that the status field is not set
if 'publication_state' in data:
raise InvalidDepositError(
'Field "publication_state" cannot be set.')
data['publication_state'] = PublicationStates.draft.name
# Set record's schema
if '$schema' in data:
raise InvalidDepositError('"$schema" field should not be set.')
# Retrieve reserved record PID which should have already been created
# by the deposit minter (The record PID value is the same
# as the one of the deposit)
rec_pid = RecordUUIDProvider.get(data['_deposit']['id']).pid
version_master, prev_version = None, None
# if this is a new version of an existing record, add the future
# record pid in the chain of versions.
if version_of:
version_master, prev_version = \
find_version_master_and_previous_record(version_of)
# The new version must be in the same community
if data['community'] != prev_version['community']:
raise ValidationError(
'The community field cannot change between versions.')
try:
version_master.insert_draft_child(rec_pid)
except Exception as exc:
# Only one draft is allowed per version chain.
if 'Draft child already exists for this relation' in \
exc.args[0]:
raise DraftExistsVersioningError(
version_master.draft_child
)
raise exc
else:
# create parent PID
parent_pid = RecordUUIDProvider.create().pid
version_master = PIDVersioning(parent=parent_pid)
version_master.insert_draft_child(child=rec_pid)
# Mint the deposit with the parent PID
data['_pid'] = [{
'value': version_master.parent.pid_value,
'type': RecordUUIDProvider.parent_pid_type,
}]
if 'community' not in data or not data['community']:
raise ValidationError(
'Record\s metadata has no community field.')
try:
community_id = uuid.UUID(data['community'])
except ValueError as e:
raise InvalidDepositError(
'Community ID is not a valid UUID.') from e
try:
schema = CommunitySchema.get_community_schema(community_id)
except CommunitySchemaDoesNotExistError as e:
raise InvalidDepositError(
'No schema for community {}.'.format(community_id)) from e
if version_of:
data['$schema'] = Deposit._build_deposit_schema(prev_version)
else:
from b2share.modules.schemas.serializers import \
community_schema_draft_json_schema_link
data['$schema'] = community_schema_draft_json_schema_link(
schema,
_external=True
)
# create file bucket
if prev_version and prev_version.files:
# Clone the bucket from the previous version. This doesn't
# duplicate files.
bucket = prev_version.files.bucket.snapshot(lock=False)
bucket.locked = False
else:
bucket = Bucket.create(storage_class=current_app.config[
'DEPOSIT_DEFAULT_STORAGE_CLASS'
])
if 'external_pids' in data:
create_b2safe_file(data['external_pids'], bucket)
del data['external_pids']
deposit = super(Deposit, cls).create(data, id_=id_)
db.session.add(bucket)
db.session.add(RecordsBuckets(
record_id=deposit.id, bucket_id=bucket.id
))
return deposit
def delete_record(record_uuid, reason, user):
"""Delete the record and it's PIDs.
:param record_uuid: UUID of the record to be removed.
:param reason: Reason for removal. Either one of: 'spam', 'uploader',
'takedown' (see 'ZENODO_REMOVAL_REASONS' variable in config),
otherwise using it as a verbatim "Reason" string.
:param user: ID or email of the Zenodo user (admin)
responsible for the removal.
"""
from invenio_github.models import ReleaseStatus
if isinstance(user, text_type):
user_id = User.query.filter_by(email=user).one().id
elif isinstance(user, int):
user_id = User.query.get(user).id
else:
raise TypeError("User cannot be determined from argument: {0}".format(
user))
record = ZenodoRecord.get_record(record_uuid)
# Remove the record from versioning and delete the recid
recid = PersistentIdentifier.get('recid', record['recid'])
pv = PIDVersioning(child=recid)
pv.remove_child(recid)
pv.update_redirect()
recid.delete()
# Remove the record from index
try:
RecordIndexer().delete(record)
except NotFoundError:
pass
# Remove buckets
record_bucket = record.files.bucket
RecordsBuckets.query.filter_by(record_id=record.id).delete()
record_bucket.locked = False
record_bucket.remove()
removal_reasons = dict(current_app.config['ZENODO_REMOVAL_REASONS'])
if reason in removal_reasons:
reason = removal_reasons[reason]
depid, deposit = deposit_resolver.resolve(record['_deposit']['id'])
try:
doi = PersistentIdentifier.get('doi', record['doi'])
except PIDDoesNotExistError:
doi = None
# Record OpenAIRE info
try:
original_id = openaire_original_id(record, openaire_type(record))[1]
datasource_id = openaire_datasource_id(record)
except PIDDoesNotExistError:
original_id = None
datasource_id = None
if pv.children.count() == 0:
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
conceptrecid.delete()
new_last_child = None
else:
new_last_child = (pv.last_child.pid_value,
str(pv.last_child.object_uuid))
if 'conceptdoi' in record:
conceptdoi_value = record['conceptdoi']
else:
conceptdoi_value = None
# Completely delete the deposit
# Deposit will be removed from index
deposit.delete(delete_published=True)
# Clear the record and put the deletion information
record.clear()
record.update({
'removal_reason': reason,
'removed_by': user_id,
})
record.commit()
# Mark the relevant GitHub Release as deleted
for ghr in record.model.github_releases:
ghr.status = ReleaseStatus.DELETED
db.session.commit()
# After successful DB commit, sync the DOIs with DataCite
datacite_inactivate.delay(doi.pid_value)
if conceptdoi_value:
if new_last_child:
# Update last child (updates also conceptdoi)
pid_value, rec_uuid = new_last_child
datacite_register.delay(pid_value, rec_uuid)
else:
datacite_inactivate.delay(conceptdoi_value)
# Also delete from OpenAIRE index
if current_app.config['OPENAIRE_DIRECT_INDEXING_ENABLED'] and original_id \
and datasource_id:
openaire_delete.delay(original_id=original_id,
datasource_id=datasource_id)