def create_sip_for_record(recid, agent=None, user_id=432):
"""Create a new SIP if the record's files diverged from last SIPFiles.
:param agent: Agent JSON passed to the SIP.
:param user_id: ID of the user resposible for the SIP
(by default, user ID of [email protected])
"""
pid = PersistentIdentifier.get('recid', recid)
rec = ZenodoRecord.get_record(pid.object_uuid)
recsip = RecordSIP.query.filter_by(pid_id=pid.id).order_by(
RecordSIP.created.desc()).first()
rec_f = sorted([f['file_id'] for f in rec.get('_files', [])])
sipfiles = recsip.sip.sip_files if recsip else []
sipfiles_s = sorted([str(s.file_id) for s in sipfiles])
default_agent = {
"$schema": "https://zenodo.org/schemas/sipstore/"
"agent-webclient-v1.0.0.json",
"ip_address": "127.0.0.1",
"email": "*****@*****.**"
}
agent = agent or default_agent
archivable = True
if sipfiles_s != rec_f:
RecordSIPApi.create(pid,
rec,
archivable,
create_sip_files=True,
agent=agent,
user_id=user_id)
db.session.commit()
def test_RecordSIP_create(db, mocker):
"""Test create method from the API class RecordSIP."""
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
# setup metadata
mtype = SIPMetadataType(title='JSON Test', name='json-test',
format='json', schema='url://to/schema')
db.session.add(mtype)
db.session.commit()
# first we create a record
recid = uuid.uuid4()
pid = PersistentIdentifier.create(
'recid',
'1337',
object_type='rec',
object_uuid=recid,
status=PIDStatus.REGISTERED)
mocker.patch('invenio_records.api.RecordBase.validate',
return_value=True, autospec=True)
record = Record.create(
{'title': 'record test', '$schema': 'url://to/schema'},
recid)
# we add a file to the record
bucket = Bucket.create()
content = b'Test file\n'
RecordsBuckets.create(record=record.model, bucket=bucket)
record.files['test.txt'] = BytesIO(content)
db.session.commit()
# Let's create a SIP
user = create_test_user('*****@*****.**')
agent = {'email': '*****@*****.**', 'ip_address': '1.1.1.1'}
rsip = RecordSIP.create(pid, record, True, user_id=user.id, agent=agent)
db.session.commit()
# test!
assert RecordSIP_.query.count() == 1
assert SIP_.query.count() == 1
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 1
assert len(rsip.sip.files) == 1
assert len(rsip.sip.metadata) == 1
metadata = rsip.sip.metadata[0]
assert metadata.type.format == 'json'
assert '"title": "record test"' in metadata.content
assert rsip.sip.archivable is True
# we try with no files
rsip = RecordSIP.create(pid, record, True, create_sip_files=False,
user_id=user.id, agent=agent)
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 2
assert len(rsip.sip.files) == 0
assert len(rsip.sip.metadata) == 1
# finalization
rmtree(tmppath)
def publish(self, pid=None, id_=None, user_id=None, sip_agent=None):
"""Publish the Zenodo deposit."""
self['owners'] = self['_deposit']['owners']
self.validate_publish()
is_first_publishing = not self.is_published()
deposit = super(ZenodoDeposit, self).publish(pid, id_)
recid, record = deposit.fetch_published()
RecordSIP.create(
recid, record, archivable=True,
create_sip_files=is_first_publishing, user_id=user_id,
agent=sip_agent)
# FIXME: Launch archiving task here! (or whatever we come up with)
return deposit
def publish(self, pid=None, id_=None, user_id=None, sip_agent=None):
"""Publish the Zenodo deposit."""
self['owners'] = self['_deposit']['owners']
self.validate_publish()
is_first_publishing = not self.is_published()
deposit = super(ZenodoDeposit, self).publish(pid, id_)
recid, record = deposit.fetch_published()
RecordSIP.create(recid,
record,
archivable=True,
create_sip_files=is_first_publishing,
user_id=user_id,
agent=sip_agent)
# FIXME: Launch archiving task here! (or whatever we come up with)
return deposit
def publish(self,
pid=None,
id_=None,
user_id=None,
sip_agent=None,
spam_check=True):
"""Publish the Zenodo deposit."""
self['owners'] = self['_deposit']['owners']
self.validate_publish()
if spam_check:
self.spam_check()
is_first_publishing = not self.is_published()
deposit = super(ZenodoDeposit, self).publish(pid, id_)
recid, record = deposit.fetch_published()
pv = PIDVersioning(child=recid)
is_new_version = pv.children.count() > 1
# a) Fetch the last SIP from the previous version if it's a new version
# b) Fetch the previous SIP if publishing the metadata edit
if is_new_version or (not is_first_publishing):
if is_new_version:
sip_recid = pv.children.all()[-2]
else: # (not is_first_publishing)
sip_recid = recid
# Get the last SIP of the relevant recid, i.e.: either last
# version or the current one
sip_patch_of = (db.session.query(SIPModel).join(
RecordSIPModel, RecordSIPModel.sip_id == SIPModel.id).filter(
RecordSIPModel.pid_id == sip_recid.id).order_by(
SIPModel.created.desc()).first())
else:
sip_patch_of = None
recordsip = RecordSIP.create(recid,
record,
archivable=True,
create_sip_files=is_first_publishing,
user_id=user_id,
agent=sip_agent)
archiver = BagItArchiver(
recordsip.sip,
include_all_previous=(not is_first_publishing),
patch_of=sip_patch_of)
archiver.save_bagit_metadata()
return deposit
def test_RecordSIP(db):
"""Test RecordSIP API class."""
user = create_test_user('*****@*****.**')
agent = {'email': '*****@*****.**', 'ip_address': '1.1.1.1'}
# we create a record
recid = uuid.uuid4()
pid = PersistentIdentifier.create('recid',
'1337',
object_type='rec',
object_uuid=recid,
status=PIDStatus.REGISTERED)
title = {'title': 'record test'}
record = Record.create(title, recid)
# we create the models
sip = SIP.create(True, user_id=user.id, agent=agent)
recordsip = RecordSIP_(sip_id=sip.id, pid_id=pid.id)
db.session.commit()
# We create an API SIP on top of it
api_recordsip = RecordSIP(recordsip, sip)
assert api_recordsip.model is recordsip
assert api_recordsip.sip.id == sip.id
def publish(self, pid=None, id_=None, user_id=None, sip_agent=None):
"""Publish the Zenodo deposit."""
self['owners'] = self['_deposit']['owners']
self.validate_publish()
is_first_publishing = not self.is_published()
deposit = super(ZenodoDeposit, self).publish(pid, id_)
recid, record = deposit.fetch_published()
pv = PIDVersioning(child=recid)
is_new_version = pv.children.count() > 1
# a) Fetch the last SIP from the previous version if it's a new version
# b) Fetch the previous SIP if publishing the metadata edit
if is_new_version or (not is_first_publishing):
if is_new_version:
sip_recid = pv.children.all()[-2]
else: # (not is_first_publishing)
sip_recid = recid
# Get the last SIP of the relevant recid, i.e.: either last
# version or the current one
sip_patch_of = (
db.session.query(SIPModel)
.join(RecordSIPModel, RecordSIPModel.sip_id == SIPModel.id)
.filter(RecordSIPModel.pid_id == sip_recid.id)
.order_by(SIPModel.created.desc())
.first()
)
else:
sip_patch_of = None
recordsip = RecordSIP.create(
recid, record, archivable=True,
create_sip_files=is_first_publishing, user_id=user_id,
agent=sip_agent)
archiver = BagItArchiver(
recordsip.sip, include_all_previous=(not is_first_publishing),
patch_of=sip_patch_of)
archiver.save_bagit_metadata()
return deposit
def test_RecordSIP_create(db, mocker):
"""Test create method from the API class RecordSIP."""
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
# setup metadata
mtype = SIPMetadataType(title='JSON Test',
name='json-test',
format='json',
schema='url://to/schema')
db.session.add(mtype)
db.session.commit()
# first we create a record
recid = uuid.uuid4()
pid = PersistentIdentifier.create('recid',
'1337',
object_type='rec',
object_uuid=recid,
status=PIDStatus.REGISTERED)
mocker.patch('invenio_records.api.RecordBase.validate',
return_value=True,
autospec=True)
record = Record.create(
{
'title': 'record test',
'$schema': 'url://to/schema'
}, recid)
# we add a file to the record
bucket = Bucket.create()
content = b'Test file\n'
RecordsBuckets.create(record=record.model, bucket=bucket)
record.files['test.txt'] = BytesIO(content)
db.session.commit()
# Let's create a SIP
user = create_test_user('*****@*****.**')
agent = {'email': '*****@*****.**', 'ip_address': '1.1.1.1'}
rsip = RecordSIP.create(pid, record, True, user_id=user.id, agent=agent)
db.session.commit()
# test!
assert RecordSIP_.query.count() == 1
assert SIP_.query.count() == 1
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 1
assert len(rsip.sip.files) == 1
assert len(rsip.sip.metadata) == 1
metadata = rsip.sip.metadata[0]
assert metadata.type.format == 'json'
assert '"title": "record test"' in metadata.content
assert rsip.sip.archivable is True
# we try with no files
rsip = RecordSIP.create(pid,
record,
True,
create_sip_files=False,
user_id=user.id,
agent=agent)
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 2
assert len(rsip.sip.files) == 0
assert len(rsip.sip.metadata) == 1
# try with specific SIP metadata type
mtype = SIPMetadataType(title='JSON Test 2',
name='json-test-2',
format='json',
schema=None) # no schema
db.session.add(mtype)
db.session.commit()
rsip = RecordSIP.create(pid,
record,
True,
create_sip_files=False,
user_id=user.id,
agent=agent,
sip_metadata_type='json-test-2')
assert SIPMetadata.query.count() == 3
assert len(rsip.sip.metadata) == 1
assert rsip.sip.metadata[0].type.id == mtype.id
# finalization
rmtree(tmppath)
'$schema':
'https://zenodo.org/schemas/deposits/records/record-v1.0.0.json',
'_deposit': {
'status': 'draft'
},
'title': 'demo'
}, recid)
record.commit()
db.session.commit()
# put a file in the record
stream = BytesIO(b'head crab\n')
b = Bucket.create()
RecordsBuckets.create(bucket=b, record=record.model)
db.session.commit()
record.files['crab.txt'] = stream
record.files.dumps()
record.commit()
db.session.commit()
# create the archive
sip = RecordSIP.create(pid,
record,
True,
user_id=1,
agent={'demo': 'archivematica'})
db.session.commit()
# archive it
oais_start_transfer(sip.sip.id, 'test-demo-archivematica')