def _publish_new(self, id_=None):
"""Publish new deposit with communities handling."""
dep_comms = set(self.pop('communities', []))
record = super(ZenodoDeposit, self)._publish_new(id_=id_)
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
if pv.children.count() > 1:
files_set = set(f.get_version().file.checksum for f in self.files)
for prev_recid in pv.children.all()[:-1]:
rec = ZenodoRecord.get_record(prev_recid.object_uuid)
prev_files_set = set(f.get_version().file.checksum for f in
rec.files)
if files_set == prev_files_set:
raise VersioningFilesError()
prev_recid = pv.children.all()[-2]
rec_comms = set(ZenodoRecord.get_record(
prev_recid.get_assigned_object()).get('communities', []))
else:
rec_comms = set()
record = self._sync_communities(dep_comms, rec_comms, record)
record.commit()
# Update the concept recid redirection
pv.update_redirect()
RecordDraft.unlink(record.pid, self.pid)
index_siblings(record.pid, neighbors_eager=True, with_deposits=True)
return record
def _publish_new(self, id_=None):
"""Publish new deposit with communities handling."""
dep_comms = set(self.pop('communities', []))
record = super(ZenodoDeposit, self)._publish_new(id_=id_)
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
if pv.children.count() > 1:
files_set = set(f.get_version().file.checksum for f in self.files)
for prev_recid in pv.children.all()[:-1]:
rec = ZenodoRecord.get_record(prev_recid.object_uuid)
prev_files_set = set(f.get_version().file.checksum for f in
rec.files)
if files_set == prev_files_set:
raise VersioningFilesError()
prev_recid = pv.children.all()[-2]
rec_comms = set(ZenodoRecord.get_record(
prev_recid.get_assigned_object()).get('communities', []))
else:
rec_comms = set()
record = self._sync_communities(dep_comms, rec_comms, record)
record.commit()
# Update the concept recid redirection
pv.update_redirect()
RecordDraft.unlink(record.pid, self.pid)
index_siblings(record.pid, neighbors_eager=True, with_deposits=True)
return record
def delete(self, delete_published=False, *args, **kwargs):
"""Delete the deposit.
:param delete_published: If True, even deposit of a published record
will be deleted (usually used by admin operations).
:type delete_published: bool
"""
is_published = self['_deposit'].get('pid')
if is_published and not delete_published:
raise PIDInvalidAction()
# Delete the recid
recid = PersistentIdentifier.get(pid_type='recid',
pid_value=self['recid'])
versioning = PIDVersioning(child=recid)
if versioning.exists:
if versioning.draft_child and \
self.pid == versioning.draft_child_deposit:
versioning.remove_draft_child()
if versioning.last_child:
index_siblings(versioning.last_child,
children=versioning.children.all(),
include_pid=True,
neighbors_eager=True,
with_deposits=True)
if recid.status == PIDStatus.RESERVED:
db.session.delete(recid)
if 'conceptrecid' in self:
concept_recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['conceptrecid'])
if concept_recid.status == PIDStatus.RESERVED:
db.session.delete(concept_recid)
# Completely remove bucket
bucket = self.files.bucket
with db.session.begin_nested():
# Remove Record-Bucket link
RecordsBuckets.query.filter_by(record_id=self.id).delete()
mp_q = MultipartObject.query_by_bucket(bucket)
# Remove multipart objects
Part.query.filter(
Part.upload_id.in_(
mp_q.with_entities(
MultipartObject.upload_id).subquery())).delete(
synchronize_session='fetch')
mp_q.delete(synchronize_session='fetch')
bucket.locked = False
bucket.remove()
depid = kwargs.get('pid', self.pid)
if depid:
depid.delete()
# NOTE: We call the parent of Deposit, invenio_records.api.Record since
# we need to completely override eveything that the Deposit.delete
# method does.
return super(Deposit, self).delete(*args, **kwargs)
def delete(self, delete_published=False, *args, **kwargs):
"""Delete the deposit.
:param delete_published: If True, even deposit of a published record
will be deleted (usually used by admin operations).
:type delete_published: bool
"""
is_published = self['_deposit'].get('pid')
if is_published and not delete_published:
raise PIDInvalidAction()
# Delete the recid
recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['recid'])
versioning = PIDVersioning(child=recid)
if versioning.exists:
if versioning.draft_child and \
self.pid == versioning.draft_child_deposit:
versioning.remove_draft_child()
if versioning.last_child:
index_siblings(versioning.last_child,
children=versioning.children.all(),
include_pid=True,
neighbors_eager=True,
with_deposits=True)
if recid.status == PIDStatus.RESERVED:
db.session.delete(recid)
if 'conceptrecid' in self:
concept_recid = PersistentIdentifier.get(
pid_type='recid', pid_value=self['conceptrecid'])
if concept_recid.status == PIDStatus.RESERVED:
db.session.delete(concept_recid)
# Completely remove bucket
bucket = self.files.bucket
with db.session.begin_nested():
# Remove Record-Bucket link
RecordsBuckets.query.filter_by(record_id=self.id).delete()
mp_q = MultipartObject.query_by_bucket(bucket)
# Remove multipart objects
Part.query.filter(
Part.upload_id.in_(mp_q.with_entities(
MultipartObject.upload_id).subquery())
).delete(synchronize_session='fetch')
mp_q.delete(synchronize_session='fetch')
bucket.locked = False
bucket.remove()
depid = kwargs.get('pid', self.pid)
if depid:
depid.delete()
# NOTE: We call the parent of Deposit, invenio_records.api.Record since
# we need to completely override eveything that the Deposit.delete
# method does.
return super(Deposit, self).delete(*args, **kwargs)
def index_versioned_record_siblings(sender, action=None, pid=None,
deposit=None):
"""Send previous version of published record for indexing."""
first_publish = (deposit.get('_deposit', {}).get('pid', {})
.get('revision_id')) == 0
if action == "publish" and first_publish:
recid_pid, _ = deposit.fetch_published()
current_app.logger.info(u'indexing siblings of {}', recid_pid)
index_siblings(recid_pid, neighbors_eager=True)
def newversion(self, pid=None):
"""Create a new version deposit."""
if not self.is_published():
raise PIDInvalidAction()
# Check that there is not a newer draft version for this record
pid, record = self.fetch_published()
pv = PIDVersioning(child=pid)
if (not pv.draft_child and
is_doi_locally_managed(record['doi'])):
with db.session.begin_nested():
# Get copy of the latest record
latest_record = ZenodoRecord.get_record(
pv.last_child.object_uuid)
data = latest_record.dumps()
# Get the communities from the last deposit
# and push those to the new version
latest_depid = PersistentIdentifier.get(
'depid', data['_deposit']['id'])
latest_deposit = ZenodoDeposit.get_record(
latest_depid.object_uuid)
last_communities = latest_deposit.get('communities', [])
owners = data['_deposit']['owners']
# TODO: Check other data that may need to be removed
keys_to_remove = (
'_deposit', 'doi', '_oai', '_files', '_buckets', '$schema')
for k in keys_to_remove:
data.pop(k, None)
# NOTE: We call the superclass `create()` method, because we
# don't want a new empty bucket, but an unlocked snapshot of
# the old record's bucket.
deposit = (super(ZenodoDeposit, self).create(data))
# Injecting owners is required in case of creating new
# version this outside of request context
deposit['_deposit']['owners'] = owners
if last_communities:
deposit['communities'] = last_communities
###
conceptrecid = PersistentIdentifier.get(
'recid', data['conceptrecid'])
recid = PersistentIdentifier.get(
'recid', str(data['recid']))
depid = PersistentIdentifier.get(
'depid', str(data['_deposit']['id']))
PIDVersioning(parent=conceptrecid).insert_draft_child(
child=recid)
RecordDraft.link(recid, depid)
# Pre-fill the Zenodo DOI to prevent the user from changing it
# to a custom DOI.
deposit['doi'] = doi_generator(recid.pid_value)
pv = PIDVersioning(child=pid)
index_siblings(pv.draft_child, neighbors_eager=True,
with_deposits=True)
with db.session.begin_nested():
# Create snapshot from the record's bucket and update data
snapshot = latest_record.files.bucket.snapshot(lock=False)
snapshot.locked = False
if 'extra_formats' in latest_record['_buckets']:
extra_formats_snapshot = \
latest_record.extra_formats.bucket.snapshot(
lock=False)
deposit['_buckets'] = {'deposit': str(snapshot.id)}
RecordsBuckets.create(record=deposit.model, bucket=snapshot)
if 'extra_formats' in latest_record['_buckets']:
deposit['_buckets']['extra_formats'] = \
str(extra_formats_snapshot.id)
RecordsBuckets.create(
record=deposit.model, bucket=extra_formats_snapshot)
deposit.commit()
return self
def newversion(self, pid=None):
"""Create a new version deposit."""
if not self.is_published():
raise PIDInvalidAction()
# Check that there is not a newer draft version for this record
pid, record = self.fetch_published()
pv = PIDVersioning(child=pid)
if (not pv.draft_child and
is_doi_locally_managed(record['doi'])):
with db.session.begin_nested():
# Get copy of the latest record
latest_record = ZenodoRecord.get_record(
pv.last_child.object_uuid)
data = latest_record.dumps()
# Get the communities from the last deposit
# and push those to the new version
latest_depid = PersistentIdentifier.get(
'depid', data['_deposit']['id'])
latest_deposit = ZenodoDeposit.get_record(
latest_depid.object_uuid)
last_communities = latest_deposit.get('communities', [])
owners = data['_deposit']['owners']
# TODO: Check other data that may need to be removed
keys_to_remove = (
'_deposit', 'doi', '_oai', '_files', '_buckets', '$schema')
for k in keys_to_remove:
data.pop(k, None)
# NOTE: We call the superclass `create()` method, because we
# don't want a new empty bucket, but an unlocked snapshot of
# the old record's bucket.
deposit = (super(ZenodoDeposit, self).create(data))
# Injecting owners is required in case of creating new
# version this outside of request context
deposit['_deposit']['owners'] = owners
if last_communities:
deposit['communities'] = last_communities
###
conceptrecid = PersistentIdentifier.get(
'recid', data['conceptrecid'])
recid = PersistentIdentifier.get(
'recid', str(data['recid']))
depid = PersistentIdentifier.get(
'depid', str(data['_deposit']['id']))
PIDVersioning(parent=conceptrecid).insert_draft_child(
child=recid)
RecordDraft.link(recid, depid)
# Pre-fill the Zenodo DOI to prevent the user from changing it
# to a custom DOI.
deposit['doi'] = doi_generator(recid.pid_value)
pv = PIDVersioning(child=pid)
index_siblings(pv.draft_child, neighbors_eager=True,
with_deposits=True)
with db.session.begin_nested():
# Create snapshot from the record's bucket and update data
snapshot = latest_record.files.bucket.snapshot(lock=False)
snapshot.locked = False
# FIXME: `snapshot.id` might not be present because we need to
# commit first to the DB.
# db.session.commit()
deposit['_buckets'] = {'deposit': str(snapshot.id)}
RecordsBuckets.create(record=deposit.model, bucket=snapshot)
deposit.commit()
return self
def versioning_link_records(recids):
"""Link several non-versioned records into one versioning scheme.
The records are linked in the order as they appear in the list, with
the first record being base for minting of the conceptdoi.
In case one of the records is already upgraded, its taken as the base
for conceptdoi instead, with preserving the requested order.
:param recids: list of recid values (strings) to link,
e.g.: ['1234','55125','51269']
:type recids: list of str
"""
recids_records = [
record_resolver.resolve(recid_val) for recid_val in recids
]
depids_deposits = [
deposit_resolver.resolve(record['_deposit']['id'])
for _, record in recids_records
]
rec_comms = sorted(
set(sum([rec.get('communities', []) for _, rec in recids_records],
[])))
dep_comms = sorted(
set(sum([dep.get('communities', []) for _, dep in depids_deposits],
[])))
upgraded = [(recid, rec) for recid, rec in recids_records
if 'conceptdoi' in rec]
# Determine the base record for versioning
if len(upgraded) == 0:
recid_v, record_v = recids_records[0]
elif len(upgraded) == 1:
recid_v, record_v = upgraded[0]
elif len(upgraded) > 1:
recid_v, record_v = upgraded[0]
child_recids = [
int(recid.pid_value)
for recid in PIDVersioning(child=recid_v).children.all()
]
i_upgraded = [int(recid.pid_value) for recid, rec in upgraded]
if set(child_recids) != set(i_upgraded):
raise Exception('Multiple upgraded records, which belong'
'to different versioning schemes.')
# Get the first record and mint the concept DOI for it
conceptdoi = zenodo_concept_doi_minter(record_v.id, record_v)
conceptrecid_v = PersistentIdentifier.get('recid',
record_v['conceptrecid'])
conceptrecid_v_val = conceptrecid_v.pid_value
pv_r1 = PIDVersioning(parent=conceptrecid_v)
children_recids = [c.pid_value for c in pv_r1.children.all()]
if not all(cr in recids for cr in children_recids):
raise Exception('Children of the already upgraded record: {0} are '
'not specified in the ordering: {1}'
''.format(children_recids, recids))
for (recid, record), (depid, deposit) in \
zip(recids_records, depids_deposits):
# Remove old versioning schemes for non-base recids
# Note: This will remove the child of the base-conceptrecid as well
# but that's OK, since it will be added again afterwards in the
# correct order.
conceptrecid = PersistentIdentifier.get('recid',
record['conceptrecid'])
pv = PIDVersioning(parent=conceptrecid)
pv.remove_child(recid)
if conceptrecid.pid_value != conceptrecid_v_val:
conceptrecid.delete()
# Update the 'conceptrecid' and 'conceptdoi' in records and deposits
record['conceptdoi'] = conceptdoi.pid_value
record['conceptrecid'] = conceptrecid_v.pid_value
record['communities'] = rec_comms
record.commit()
deposit['conceptdoi'] = conceptdoi.pid_value
deposit['conceptrecid'] = conceptrecid_v.pid_value
deposit['communities'] = dep_comms
deposit.commit()
# Add the child to the new versioning scheme
pv_r1.insert_child(recid)
pv_r1.update_redirect()
db.session.commit()
conceptrecid_v = PersistentIdentifier.get('recid', conceptrecid_v_val)
pv = PIDVersioning(parent=conceptrecid_v)
if current_app.config['DEPOSIT_DATACITE_MINTING_ENABLED']:
datacite_register.delay(pv.last_child.pid_value,
str(pv.last_child.object_uuid))
index_siblings(pv.last_child, with_deposits=True, eager=True)
def versioning_github_repository(uuid):
"""
Migrate the GitHub repositories.
:param uuid: UUID of the repository (invenio_github.models.Repository)
"""
from invenio_github.models import Repository, ReleaseStatus
from zenodo.modules.deposit.minters import zenodo_concept_recid_minter
from zenodo.modules.records.minters import zenodo_concept_doi_minter
from invenio_pidrelations.contrib.records import index_siblings
repository = Repository.query.get(uuid)
published_releases = repository.releases.filter_by(
status=ReleaseStatus.PUBLISHED).all()
# Nothing to migrate if no successful release was ever made
if not published_releases:
return
deposits = [
ZenodoDeposit.get_record(r.record_id) for r in published_releases
if r.recordmetadata.json is not None
]
deposits = [dep for dep in deposits if 'removed_by' not in dep]
deposits = sorted(deposits, key=lambda dep: int(dep['recid']))
recids = [
PersistentIdentifier.get('recid', dep['recid']) for dep in deposits
]
records = [ZenodoRecord.get_record(p.object_uuid) for p in recids]
# There were successful releases, but deposits/records were removed since
if not records:
return
assert not any('conceptrecid' in rec for rec in records), \
"One or more of the release records have been already migrated"
assert not any('conceptrecid' in dep for dep in deposits), \
"One or more of the release deposits have been already migrated"
conceptrecid = zenodo_concept_recid_minter(record_uuid=records[0].id,
data=records[0])
conceptrecid.register()
# Mint the Concept DOI if we are migrating (linking) more than one record
if len(records) > 1:
conceptdoi = zenodo_concept_doi_minter(records[0].id, records[0])
else:
conceptdoi = None
rec_comms = sorted(
set(sum([rec.get('communities', []) for rec in records], [])))
dep_comms = sorted(
set(sum([dep.get('communities', []) for dep in deposits], [])))
for rec in records:
rec['conceptrecid'] = conceptrecid.pid_value
if conceptdoi:
rec['conceptdoi'] = conceptdoi.pid_value
if rec_comms:
rec['communities'] = rec_comms
rec.commit()
for dep in deposits:
dep['conceptrecid'] = conceptrecid.pid_value
if conceptdoi:
dep['conceptdoi'] = conceptdoi.pid_value
if dep_comms:
dep['communities'] = dep_comms
dep.commit()
pv = PIDVersioning(parent=conceptrecid)
for recid in recids:
pv.insert_child(recid)
pv.update_redirect()
if current_app.config['DEPOSIT_DATACITE_MINTING_ENABLED']:
datacite_register.delay(recids[-1].pid_value, str(records[-1].id))
db.session.commit()
# Reindex all siblings
index_siblings(pv.last_child, with_deposits=True)
