def test_Archive(db):
"""Test the Archive model class."""
assert Archive.query.count() == 0
# we create an SIP, it will automatically create an Archive via signals
user = create_test_user('*****@*****.**')
sip = SIP.create(True, user_id=user.id, agent={'test': 'test'})
db.session.commit()
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.sip.user.id == sip.user.id
assert ark.status == ArchiveStatus.NEW
assert ark.accession_id is None
assert ark.archivematica_id is None
# let's change the object
ark.status = ArchiveStatus.REGISTERED
ark.accession_id = '08'
ark.archivematica_id = sip.id
db.session.commit()
ark = Archive.get_from_accession_id('08')
assert Archive.query.count() == 1
assert ark.status == ArchiveStatus.REGISTERED
assert ark.archivematica_id == sip.id
# we try to get a non existing record
assert Archive.get_from_sip(uuid.uuid4()) is None
def oais_start_transfer(uuid, accession_id='', archivematica_id=None):
"""Archive a sip.
This function should be called to start a transfer to archive a sip.
Once the transfer is finished, you should call
:py:func:`invenio_archivematica.tasks.oais_finish_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_started`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip to archive
:param str accession_id: the AIP accession ID. You can generate one from
:py:func:`invenio_archivematica.factories.create_accession_id`
"""
# we get the sip
sip = SIP.get_sip(uuid)
# we register the sip as being processed
ark = Archive.get_from_sip(uuid)
if not ark:
ark = Archive.create(sip.model)
ark.accession_id = accession_id
ark.status = ArchiveStatus.WAITING
# we start the transfer
imp = current_app.config['ARCHIVEMATICA_TRANSFER_FACTORY']
transfer = import_string(imp)
ret = transfer(sip.id, current_app.config['ARCHIVEMATICA_TRANSFER_FOLDER'])
if ret == 0:
db.session.commit()
oais_transfer_started.send(sip)
return
oais_fail_transfer(uuid, accession_id)
def transfer_demo(uuid, config):
"""Transfer the files contained in the sip to the destination.
Very similar to the rsync transfer. However, because of time, I use the
VERY UNSECURE sshpass package for rsync authentication.
DO NOT USE IN PROD!!!
:param str uuid: the id of the sip containing files to transfer
:param dict config: here config must be a dict with the following keys:
- user - the SSH user
- password_file - a path where the password is stored
- remote - the URL or IP of the remote
- remote_path - where to store files on the remote
- args - the args for rsync
"""
# we retrieve the archive and the SIP associated
sip = SIP.get_sip(uuid)
ark = Archive.get_from_sip(uuid)
# we export it to the temp folder
archiver = BaseArchiver(sip)
archiver.write_all_files()
# we rsync it to the remote
src_path = archiver.get_fullpath('')
dest_path = join(config['remote_path'], ark.accession_id)
dest_path = '{}:{}'.format(config['remote'], dest_path)
ssh_command = 'sshpass -f {filename} ssh -l {user}'.format(
filename=config['password_file'], user=config['user'])
return call([
'rsync', config['args'], '--rsh={}'.format(ssh_command), src_path,
dest_path
])
def test_oais_fail_transfer(db):
"""Test the oais_fail_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
# we fail the transfer
oais_fail_transfer(sip.id)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.FAILED
def test_oais_process_transfer(db):
"""Test the oais_process_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
aipid = uuid.uuid4()
oais_process_transfer(sip.id, archivematica_id=aipid)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.PROCESSING_TRANSFER
assert ark.archivematica_id == aipid
def test_oais_finish_transfer(db):
"""Test the oais_finish_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
aipid = uuid.uuid4()
db.session.commit()
# we finish the transfer
oais_finish_transfer(sip.id, archivematica_id=aipid)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.REGISTERED
assert ark.archivematica_id == aipid
assert ark.sip.archived is True
def archive_directory_builder(sip):
"""Build a directory structure for the archived SIP.
Creates a structure that is based on the Archive object linked to the SIP.
It takes its accession_id. In case no Archive object exists, it returns
the ID of the SIP.
:param sip: SIP which is to be archived
:type SIP: invenio_sipstore.models.SIP
:returns: list of str
"""
ark = Archive.get_from_sip(sip.id)
if not ark and not ark.accession_id:
return [str(sip.id)]
return [ark.accession_id]
def test_oais_start_transfer(app, db, location):
"""Test the oais_start_transfer function."""
assert Archive.query.count() == 0
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
assert Archive.query.count() == 1
# we start the transfer
oais_start_transfer(sip.id, '1991')
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.WAITING
assert ark.accession_id == '1991'
# we try the case where no archive exist and transfer fails
db.session.delete(ark)
db.session.commit()
app.config['ARCHIVEMATICA_TRANSFER_FACTORY'] = 'helpers:transfer_fail'
assert Archive.query.count() == 0
oais_start_transfer(sip.id, '1991')
ark = Archive.get_from_sip(sip.id)
assert Archive.query.count() == 1
assert ark.status == ArchiveStatus.FAILED
assert ark.accession_id == '1991'
assert ark.sip.archived is False
def test_listeners(conf, expected_status, app, db):
"""Test listener_sip_created and listener_record_updated functions."""
# first we change the is_archivable function
app.config['ARCHIVEMATICA_ISARCHIVABLE_FACTORY'] = conf
assert Archive.query.count() == 0
# let's create an SIP
user = create_test_user('*****@*****.**')
sip = SIP.create(True, user_id=user.id, agent={'test': 'test'})
db.session.commit()
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.sip.user.id == sip.user.id
assert ark.status == expected_status
def oais_fail_transfer(uuid, accession_id='', archivematica_id=None):
"""Mark the transfer as failed.
This function should be called if the transfer failed. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_failed`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.FAILED
ark.sip.archived = False
db.session.commit()
oais_transfer_failed.send(SIP(ark.sip))
def oais_process_aip(uuid, accession_id='', archivematica_id=None):
"""Mark the aip in progress.
This function should be called if the aip is processing. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal
:py:data:`invenio_archivematica.signals.oais_transfer_processing`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
:param str archivematica_id: the ID of the AIP in Archivematica
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.PROCESSING_AIP
ark.archivematica_id = archivematica_id
db.session.commit()
oais_transfer_processing.send(SIP(ark.sip))
def oais_finish_transfer(uuid, accession_id='', archivematica_id=None):
"""Finalize the transfer of a sip.
This function should be called once the transfer has been finished, to
mark the sip as correctly archived. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_finished`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
:param str archivematica_id: the ID in Archivematica of the created AIP
(should be an UUID)
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.REGISTERED
ark.archivematica_id = archivematica_id
ark.sip.archived = True
db.session.commit()
oais_transfer_finished.send(SIP(ark.sip))