def oais_start_transfer(uuid, accession_id='', archivematica_id=None):
"""Archive a sip.
This function should be called to start a transfer to archive a sip.
Once the transfer is finished, you should call
:py:func:`invenio_archivematica.tasks.oais_finish_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_started`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip to archive
:param str accession_id: the AIP accession ID. You can generate one from
:py:func:`invenio_archivematica.factories.create_accession_id`
"""
# we get the sip
sip = SIP.get_sip(uuid)
# we register the sip as being processed
ark = Archive.get_from_sip(uuid)
if not ark:
ark = Archive.create(sip.model)
ark.accession_id = accession_id
ark.status = ArchiveStatus.WAITING
# we start the transfer
imp = current_app.config['ARCHIVEMATICA_TRANSFER_FACTORY']
transfer = import_string(imp)
ret = transfer(sip.id, current_app.config['ARCHIVEMATICA_TRANSFER_FOLDER'])
if ret == 0:
db.session.commit()
oais_transfer_started.send(sip)
return
oais_fail_transfer(uuid, accession_id)
def test_Archive(db):
"""Test the Archive model class."""
assert Archive.query.count() == 0
# we create an SIP, it will automatically create an Archive via signals
user = create_test_user('*****@*****.**')
sip = SIP.create(True, user_id=user.id, agent={'test': 'test'})
db.session.commit()
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.sip.user.id == sip.user.id
assert ark.status == ArchiveStatus.NEW
assert ark.accession_id is None
assert ark.archivematica_id is None
# let's change the object
ark.status = ArchiveStatus.REGISTERED
ark.accession_id = '08'
ark.archivematica_id = sip.id
db.session.commit()
ark = Archive.get_from_accession_id('08')
assert Archive.query.count() == 1
assert ark.status == ArchiveStatus.REGISTERED
assert ark.archivematica_id == sip.id
# we try to get a non existing record
assert Archive.get_from_sip(uuid.uuid4()) is None
def test_oais_fail_transfer(db):
"""Test the oais_fail_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
# we fail the transfer
oais_fail_transfer(sip.id)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.FAILED
def test_oais_process_transfer(db):
"""Test the oais_process_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
aipid = uuid.uuid4()
oais_process_transfer(sip.id, archivematica_id=aipid)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.PROCESSING_TRANSFER
assert ark.archivematica_id == aipid
def test_oais_finish_transfer(db):
"""Test the oais_finish_transfer function."""
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
aipid = uuid.uuid4()
db.session.commit()
# we finish the transfer
oais_finish_transfer(sip.id, archivematica_id=aipid)
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.REGISTERED
assert ark.archivematica_id == aipid
assert ark.sip.archived is True
def test_Archive_get_realstatus_transfer(db, client, oauth2):
"""Test the Archive's get method with transfer processing."""
sip = SIP.create()
ark = Archive.create(sip=sip,
accession_id='id',
archivematica_id=uuid.uuid4())
ark.status = ArchiveStatus.WAITING
db.session.commit()
mock_response = Response()
mock_response.status_code = 200
mock_response._content = json.dumps({
'status': 'SIP_PROCESSING'
}).encode('utf-8')
with patch('requests.get', return_value=mock_response):
response = client.get(url_for('invenio_archivematica_api.archive_api',
accession_id=ark.accession_id,
access_token=oauth2.token),
data=json.dumps({'realStatus': True}),
content_type='application/json')
assert response.status_code == 200
result = json.loads(response.data.decode('utf-8'))
assert 'sip_id' in result and result['sip_id'] == str(sip.id)
assert 'status' in result and result['status'] == 'PROCESSING_TRANSFER'
assert 'accession_id' in result and result['accession_id'] == 'id'
assert 'archivematica_id' in result \
and result['archivematica_id'] == str(ark.archivematica_id)
def transfer_demo(uuid, config):
"""Transfer the files contained in the sip to the destination.
Very similar to the rsync transfer. However, because of time, I use the
VERY UNSECURE sshpass package for rsync authentication.
DO NOT USE IN PROD!!!
:param str uuid: the id of the sip containing files to transfer
:param dict config: here config must be a dict with the following keys:
- user - the SSH user
- password_file - a path where the password is stored
- remote - the URL or IP of the remote
- remote_path - where to store files on the remote
- args - the args for rsync
"""
# we retrieve the archive and the SIP associated
sip = SIP.get_sip(uuid)
ark = Archive.get_from_sip(uuid)
# we export it to the temp folder
archiver = BaseArchiver(sip)
archiver.write_all_files()
# we rsync it to the remote
src_path = archiver.get_fullpath('')
dest_path = join(config['remote_path'], ark.accession_id)
dest_path = '{}:{}'.format(config['remote'], dest_path)
ssh_command = 'sshpass -f {filename} ssh -l {user}'.format(
filename=config['password_file'], user=config['user'])
return call([
'rsync', config['args'], '--rsh={}'.format(ssh_command), src_path,
dest_path
])
def listener_sip_created(sip, *args, **kwargs):
"""Create an entry in the database when a sip is created."""
imp = current_app.config['ARCHIVEMATICA_ISARCHIVABLE_FACTORY']
is_archivable = import_string(imp) if imp else None
ark = Archive.create(sip.model)
if not is_archivable or not is_archivable(sip):
ark.status = ArchiveStatus.IGNORED
def test_create_accessioned_id(db):
"""Test ``create_accessioned_id`` function."""
# First, we create a SIP
sip = SIP.create()
ark = Archive.create(sip)
db.session.commit()
accessioned_id = factories.create_accession_id(ark)
assert accessioned_id == 'CERN-' + str(sip.id)
def test_create_accessioned_id(db):
"""Test ``create_accessioned_id`` function."""
# First, we create a SIP
sip = SIP.create()
ark = Archive.create(sip)
db.session.commit()
accessioned_id = factories.create_accession_id(ark)
assert accessioned_id \
== current_app.config['ARCHIVEMATICA_ORGANIZATION_NAME'] + '-' \
+ str(sip.id)
def test_ArchiveDownload_get_412(db, client, oauth2):
"""Test the Download's get method with no archivematica_id."""
sip = SIP.create()
ark = Archive.create(sip=sip, accession_id='id')
db.session.commit()
response = client.get(
url_for('invenio_archivematica_api.download_api',
accession_id=ark.accession_id,
access_token=oauth2.token))
assert response.status_code == 412
def test_ArchiveDownload_get_520(db, client, oauth2):
"""Test the Download's get method with no storage server running."""
sip = SIP.create()
ark = Archive.create(sip=sip,
accession_id='id',
archivematica_id=uuid.uuid4())
ark.status = ArchiveStatus.REGISTERED
db.session.commit()
response = client.get(
url_for('invenio_archivematica_api.download_api',
accession_id=ark.accession_id,
access_token=oauth2.token))
assert response.status_code == 520
def archive_directory_builder(sip):
"""Build a directory structure for the archived SIP.
Creates a structure that is based on the Archive object linked to the SIP.
It takes its accession_id. In case no Archive object exists, it returns
the ID of the SIP.
:param sip: SIP which is to be archived
:type SIP: invenio_sipstore.models.SIP
:returns: list of str
"""
ark = Archive.get_from_sip(sip.id)
if not ark and not ark.accession_id:
return [str(sip.id)]
return [ark.accession_id]
def test_archive_new_sips(db, location):
"""Test the archive_new_sips function."""
# we create 2 SIP
sip1 = SIP.create()
Archive.create(sip1)
db.session.commit()
time.sleep(3)
sip2 = SIP.create()
Archive.create(sip2)
db.session.commit()
# we archive all records older than 2 seconds
archive_new_sips('invenio_archivematica.factories.create_accession_id',
days=0,
seconds=2,
delay=False)
arks = Archive.query.all()
assert len(arks) == 2
for ark in arks:
if ark.sip_id == sip1.id:
assert ark.status == ArchiveStatus.WAITING
# we update the archive so it will be ignored in what follows
ark.status = ArchiveStatus.IGNORED
db.session.commit()
else:
assert ark.status == ArchiveStatus.NEW
# now we archive everything, but rec2 shouldn't be archived as it is
# flagged as IGNORED
archive_new_sips('invenio_archivematica.factories.create_accession_id',
days=0,
delay=False)
arks = Archive.query.all()
assert len(arks) == 2
for ark in arks:
if ark.sip_id == sip1.id:
assert ark.status == ArchiveStatus.IGNORED
else:
assert ark.status == ArchiveStatus.WAITING
def test_listeners(conf, expected_status, app, db):
"""Test listener_sip_created and listener_record_updated functions."""
# first we change the is_archivable function
app.config['ARCHIVEMATICA_ISARCHIVABLE_FACTORY'] = conf
assert Archive.query.count() == 0
# let's create an SIP
user = create_test_user('*****@*****.**')
sip = SIP.create(True, user_id=user.id, agent={'test': 'test'})
db.session.commit()
assert Archive.query.count() == 1
ark = Archive.get_from_sip(sip.id)
assert ark.sip.user.id == sip.user.id
assert ark.status == expected_status
def test_oais_start_transfer(app, db, location):
"""Test the oais_start_transfer function."""
assert Archive.query.count() == 0
# let's create a SIP
sip = SIP.create()
Archive.create(sip)
db.session.commit()
assert Archive.query.count() == 1
# we start the transfer
oais_start_transfer(sip.id, '1991')
ark = Archive.get_from_sip(sip.id)
assert ark.status == ArchiveStatus.WAITING
assert ark.accession_id == '1991'
# we try the case where no archive exist and transfer fails
db.session.delete(ark)
db.session.commit()
app.config['ARCHIVEMATICA_TRANSFER_FACTORY'] = 'helpers:transfer_fail'
assert Archive.query.count() == 0
oais_start_transfer(sip.id, '1991')
ark = Archive.get_from_sip(sip.id)
assert Archive.query.count() == 1
assert ark.status == ArchiveStatus.FAILED
assert ark.accession_id == '1991'
assert ark.sip.archived is False
def test_ArchiveDownload_get_status_code(db, client, oauth2):
"""Test the API request for Download's get method."""
sip = SIP.create()
ark = Archive.create(sip=sip,
accession_id='id',
archivematica_id=uuid.uuid4())
ark.status = ArchiveStatus.REGISTERED
db.session.commit()
mock_response = Response()
mock_response.status_code = 404
with patch('requests.get', return_value=mock_response):
response = client.get(
url_for('invenio_archivematica_api.download_api',
accession_id=ark.accession_id,
access_token=oauth2.token))
assert response.status_code == mock_response.status_code
def test_Archive_get_status_code(db, client, oauth2):
"""Test the Archive's get method with error on Archivematica."""
sip = SIP.create()
ark = Archive.create(sip=sip,
accession_id='id',
archivematica_id=uuid.uuid4())
ark.status = ArchiveStatus.WAITING
db.session.commit()
mock_response = Response()
mock_response.status_code = 404
with patch('requests.get', return_value=mock_response):
response = client.get(url_for('invenio_archivematica_api.archive_api',
accession_id=ark.accession_id,
access_token=oauth2.token),
data=json.dumps({'realStatus': True}),
content_type='application/json')
assert response.status_code == mock_response.status_code
def oais_fail_transfer(uuid, accession_id='', archivematica_id=None):
"""Mark the transfer as failed.
This function should be called if the transfer failed. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_failed`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.FAILED
ark.sip.archived = False
db.session.commit()
oais_transfer_failed.send(SIP(ark.sip))
def test_Archive_get_200(db, client, oauth2):
"""Test the Archive's get method with no archivematica_id."""
sip = SIP.create()
ark = Archive.create(sip=sip,
accession_id='id',
archivematica_id=uuid.uuid4())
db.session.commit()
response = client.get(
url_for('invenio_archivematica_api.archive_api',
accession_id=ark.accession_id,
access_token=oauth2.token))
assert response.status_code == 200
result = json.loads(response.data.decode('utf-8'))
assert 'sip_id' in result and result['sip_id'] == str(sip.id)
assert 'status' in result and result['status'] == 'NEW'
assert 'accession_id' in result and result['accession_id'] == 'id'
assert 'archivematica_id' in result \
and result['archivematica_id'] == str(ark.archivematica_id)
def oais_process_aip(uuid, accession_id='', archivematica_id=None):
"""Mark the aip in progress.
This function should be called if the aip is processing. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal
:py:data:`invenio_archivematica.signals.oais_transfer_processing`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
:param str archivematica_id: the ID of the AIP in Archivematica
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.PROCESSING_AIP
ark.archivematica_id = archivematica_id
db.session.commit()
oais_transfer_processing.send(SIP(ark.sip))
def oais_finish_transfer(uuid, accession_id='', archivematica_id=None):
"""Finalize the transfer of a sip.
This function should be called once the transfer has been finished, to
mark the sip as correctly archived. See
:py:func:`invenio_archivematica.tasks.oais_start_transfer`.
The signal :py:data:`invenio_archivematica.signals.oais_transfer_finished`
is called with the sip as function parameter.
:param str uuid: the UUID of the sip
:param str archivematica_id: the ID in Archivematica of the created AIP
(should be an UUID)
"""
ark = Archive.get_from_sip(uuid)
ark.status = ArchiveStatus.REGISTERED
ark.archivematica_id = archivematica_id
ark.sip.archived = True
db.session.commit()
oais_transfer_finished.send(SIP(ark.sip))
def test_Archive_patch_200(db, client, oauth2):
"""Test the Archive's get method with no archivematica_id."""
sip = SIP.create()
ark = Archive.create(sip=sip, accession_id='id')
db.session.commit()
params = {'archivematica_id': str(uuid.uuid4()), 'status': 'COMPLETE'}
response = client.patch(url_for('invenio_archivematica_api.archive_api',
accession_id=ark.accession_id,
access_token=oauth2.token),
data=json.dumps(params),
content_type='application/json')
assert response.status_code == 200
result = json.loads(response.data.decode('utf-8'))
assert 'sip_id' in result and result['sip_id'] == str(sip.id)
assert 'status' in result and result['status'] == 'REGISTERED'
assert 'accession_id' in result and result['accession_id'] == 'id'
assert 'archivematica_id' in result \
and result['archivematica_id'] == params['archivematica_id']
ark = Archive.query.one()
assert ark.status == ArchiveStatus.REGISTERED
assert str(ark.archivematica_id) == params['archivematica_id']
def test(accession_id):
"""Show a test page."""
ark = Archive.get_from_accession_id(accession_id)
return """<DOCTYPE html><html><head></head><body>
<h1>{}</h1>
</body></html>""".format(create_accession_id(ark))
def decorate(*args, **kwargs):
accession_id = kwargs.pop('accession_id')
archive = Archive_.get_from_accession_id(accession_id)
if not archive:
abort(404, 'Accession_id {} not found.'.format(accession_id))
return f(archive=archive, *args, **kwargs)