def archive_sip(sip_uuid):
"""Send the SIP for archiving.
Retries every 4 hours, six times, which should work for up to 24 hours
archiving system downtime.
:param sip_uuid: UUID of the SIP for archiving.
:type sip_uuid: str
"""
try:
sip = SIPApi(SIP.query.get(sip_uuid))
archiver = BagItArchiver(sip)
bagmeta = archiver.get_bagit_metadata(sip)
if bagmeta is None:
raise ArchivingError(
'Bagit metadata does not exist for SIP: {0}.'.format(sip.id))
if sip.archived:
raise ArchivingError(
'SIP was already archived {0}.'.format(sip.id))
archiver.write_all_files()
sip.archived = True
db.session.commit()
except Exception as exc:
# On ArchivingError (see above), do not retry, but re-raise
if not isinstance(exc, ArchivingError):
archive_sip.retry(exc=exc)
raise
def test_write_all_files(sips, archive_fs):
"""Test the functions used to create an export of the SIP."""
sip = sips[0]
archiver = BagItArchiver(sip)
assert not len(archive_fs.listdir())
archiver.write_all_files()
assert len(archive_fs.listdir()) == 1
fs = archive_fs.opendir(archiver.get_archive_subpath())
assert set(fs.listdir()) == \
set(['tagmanifest-md5.txt', 'bagit.txt', 'manifest-md5.txt',
'bag-info.txt', 'data', ])
assert set(fs.listdir('data')) == \
set(['metadata', 'files', 'filenames.txt'])
assert set(fs.listdir('data/metadata')) == \
set(['marcxml-test.xml', 'json-test.json', ])
assert set(fs.listdir('data/files')) == set(['foobar.txt', ])
def test_write_patched(mocker, sips, archive_fs,
secure_sipfile_name_formatter):
"""Test the BagIt archiving with previous SIP as a base."""
# Mock the bagging date generation so the 'Bagging-Date' tag is predefined
dt = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")
mocker.patch('invenio_sipstore.archivers.bagit_archiver.BagItArchiver.'
'_generate_bagging_date', return_value=dt)
arch1 = BagItArchiver(sips[0])
arch1.write_all_files()
arch2 = BagItArchiver(sips[1], patch_of=sips[0])
arch2.write_all_files()
arch3 = BagItArchiver(sips[2], patch_of=sips[1],
include_all_previous=True)
arch3.write_all_files()
arch5 = BagItArchiver(sips[4], patch_of=sips[2],
include_all_previous=True)
arch5.write_all_files()
# NOTE: We take only SIP-1, SIP-2, SIP-3 and SIP-5.
# Enumeration of related objects follows the "sips" fixture naming
fs1 = archive_fs.opendir(arch1.get_archive_subpath())
fs2 = archive_fs.opendir(arch2.get_archive_subpath())
fs3 = archive_fs.opendir(arch3.get_archive_subpath())
fs5 = archive_fs.opendir(arch5.get_archive_subpath())
assert len(fs1.listdir()) == 5
assert len(fs2.listdir()) == 6 # Includes 'fetch.txt'
assert len(fs3.listdir()) == 6 # Includes 'fetch.txt'
assert len(fs5.listdir()) == 6 # Includes 'fetch.txt'
# Check SIP-1,2,3,5 data contents
assert set(fs1.listdir('data')) == \
set(['files', 'metadata', 'filenames.txt'])
assert len(fs1.listdir('data/files')) == 1
assert len(fs1.listdir('data/metadata')) == 2
assert set(fs2.listdir('data')) == \
set(['files', 'metadata', 'filenames.txt'])
assert len(fs2.listdir('data/files')) == 1
assert len(fs2.listdir('data/metadata')) == 2
assert set(fs3.listdir('data')) == \
set(['files', 'metadata', 'filenames.txt'])
assert len(fs3.listdir('data/files')) == 1
assert len(fs3.listdir('data/metadata')) == 2
assert set(fs5.listdir('data')) == \
set(['metadata', 'filenames.txt'])
assert len(fs5.listdir('data/metadata')) == 1
# Fetch the filenames for easier fixture formatting below
file1_fn = '{0}-foobar.txt'.format(
fetch_file_endswith(sips[0], 'foobar.txt').file_id)
file2_fn = '{0}-foobar2.txt'.format(
fetch_file_endswith(sips[1], 'foobar2.txt').file_id)
file3_fn = '{0}-foobar3.txt'.format(
fetch_file_endswith(sips[2], 'foobar3.txt').file_id)
file2_rn_fn = '{0}-foobar2-renamed.txt'.format(
fetch_file_endswith(sips[2], 'foobar2-renamed.txt').file_id)
assert file2_fn[:36] == file2_rn_fn[:36]
# Both file2_fn and file2_rn_fn are referring to the same FileInstance,
# so their UUID prefix should match
expected_sip1 = [
('data/files/{0}'.format(file1_fn), 'test'),
('data/metadata/marcxml-test.xml', '<p>XML 1</p>'),
('data/metadata/json-test.json', '{"title": "JSON 1"}'),
('bagit.txt',
'BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8'),
('manifest-md5.txt', set([
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/files/{0}'.format(file1_fn))),
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/metadata/marcxml-test.xml')),
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/metadata/json-test.json')),
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/filenames.txt')),
])),
('data/filenames.txt', set([
'{0} foobar.txt'.format(file1_fn),
])),
('bag-info.txt', (
"Source-Organization: European Organization for Nuclear Research\n"
"Organization-Address: CERN, CH-1211 Geneva 23, Switzerland\n"
"Bagging-Date: {0}\n".format(dt) +
"Payload-Oxum: 93.4\n"
"External-Identifier: {0}/SIPBagIt-v1.0.0\n".format(sips[0].id) +
"External-Description: BagIt archive of SIP."
)),
]
expected_sip2 = [
('data/files/{0}'.format(file2_fn), 'test-second'),
('data/metadata/marcxml-test.xml', '<p>XML 2</p>'),
('data/metadata/json-test.json', '{"title": "JSON 2"}'),
('bagit.txt',
'BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8'),
('fetch.txt', set(["{0} {1} {2}".format(
fs1.getsyspath('data/files/{0}'.format(file1_fn)),
4, 'data/files/{0}'.format(file1_fn)),
])),
('manifest-md5.txt', set([
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/files/{0}'.format(file1_fn))),
"{checksum} {filepath}".format(
**_read_file(fs2, 'data/files/{0}'.format(file2_fn))),
"{checksum} {filepath}".format(
**_read_file(fs2, 'data/metadata/marcxml-test.xml')),
"{checksum} {filepath}".format(
**_read_file(fs2, 'data/metadata/json-test.json')),
"{checksum} {filepath}".format(
**_read_file(fs2, 'data/filenames.txt')),
])),
('data/filenames.txt', set([
'{0} foobar.txt'.format(file1_fn),
'{0} foobar2.txt'.format(file2_fn),
])),
('bag-info.txt', (
"Source-Organization: European Organization for Nuclear Research\n"
"Organization-Address: CERN, CH-1211 Geneva 23, Switzerland\n"
"Bagging-Date: {0}\n".format(dt) +
"Payload-Oxum: 165.5\n"
"External-Identifier: {0}/SIPBagIt-v1.0.0\n".format(sips[1].id) +
"External-Description: BagIt archive of SIP."
)),
]
expected_sip3 = [
('data/files/{0}'.format(file3_fn), 'test-third'),
('data/metadata/marcxml-test.xml', '<p>XML 3</p>'),
('data/metadata/json-test.json', '{"title": "JSON 3"}'),
('bagit.txt',
'BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8'),
('fetch.txt', set([
"{0} {1} {2}".format(
fs1.getsyspath('data/files/{0}'.format(file1_fn)),
4, 'data/files/{0}'.format(file1_fn)),
# Explanation on entry below: The file is fetched using original
# filename (file2_fn) as it will be archived in SIP-2, however
# the new destination has the 'renamed' filename (file2_rn_fn).
# This is correct and expected behaviour
"{0} {1} {2}".format(
fs2.getsyspath('data/files/{0}'.format(file2_fn)),
11, 'data/files/{0}'.format(file2_rn_fn)),
])),
('manifest-md5.txt', set([
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/files/{0}'.format(file1_fn))),
# Manifest also specifies the renamed filename for File-2
"{checksum} data/files/{newfilename}".format(
newfilename=file2_rn_fn,
**_read_file(fs2, 'data/files/{0}'.format(file2_fn))),
"{checksum} {filepath}".format(
**_read_file(fs3, 'data/files/{0}'.format(file3_fn))),
"{checksum} {filepath}".format(
**_read_file(fs3, 'data/metadata/marcxml-test.xml')),
"{checksum} {filepath}".format(
**_read_file(fs3, 'data/metadata/json-test.json')),
"{checksum} {filepath}".format(
**_read_file(fs3, 'data/filenames.txt')),
])),
('data/filenames.txt', set([
'{0} foobar.txt'.format(file1_fn),
'{0} foobar2.txt'.format(file2_fn),
'{0} foobar3.txt'.format(file3_fn),
])),
('bag-info.txt', (
"Source-Organization: European Organization for Nuclear Research\n"
"Organization-Address: CERN, CH-1211 Geneva 23, Switzerland\n"
"Bagging-Date: {0}\n".format(dt) +
"Payload-Oxum: 236.6\n"
"External-Identifier: {0}/SIPBagIt-v1.0.0\n".format(sips[2].id) +
"External-Description: BagIt archive of SIP."
)),
]
expected_sip5 = [
('data/metadata/marcxml-test.xml', '<p>XML 5 Meta Only</p>'),
('bagit.txt',
'BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8'),
('fetch.txt', set([
"{0} {1} {2}".format(
fs1.getsyspath('data/files/{0}'.format(file1_fn)),
4, 'data/files/{0}'.format(file1_fn)),
# As in "expected_sip3" above, the file is fetched using original
# filename (file2_fn) as it will be archived in SIP-2, however
# the new destination has the 'renamed' filename (file2_rn_fn).
# This is correct and expected behaviour
"{0} {1} {2}".format(
fs2.getsyspath('data/files/{0}'.format(file2_fn)),
11, 'data/files/{0}'.format(file2_rn_fn)),
"{0} {1} {2}".format(
fs3.getsyspath('data/files/{0}'.format(file3_fn)),
10, 'data/files/{0}'.format(file3_fn)),
])),
('manifest-md5.txt', set([
"{checksum} {filepath}".format(
**_read_file(fs1, 'data/files/{0}'.format(file1_fn))),
# Manifest also specifies the renamed filename for File-2
"{checksum} data/files/{newfilename}".format(
newfilename=file2_rn_fn,
**_read_file(fs2, 'data/files/{0}'.format(file2_fn))),
"{checksum} {filepath}".format(
**_read_file(fs3, 'data/files/{0}'.format(file3_fn))),
"{checksum} {filepath}".format(
**_read_file(fs5, 'data/metadata/marcxml-test.xml')),
"{checksum} {filepath}".format(
**_read_file(fs5, 'data/filenames.txt')),
])),
('data/filenames.txt', set([
'{0} foobar.txt'.format(file1_fn),
'{0} foobar2.txt'.format(file2_fn),
'{0} foobar3.txt'.format(file3_fn),
])),
('bag-info.txt', (
"Source-Organization: European Organization for Nuclear Research\n"
"Organization-Address: CERN, CH-1211 Geneva 23, Switzerland\n"
"Bagging-Date: {0}\n".format(dt) +
"Payload-Oxum: 227.5\n"
"External-Identifier: {0}/SIPBagIt-v1.0.0\n".format(sips[4].id) +
"External-Description: BagIt archive of SIP."
)),
]
for fs, expected in [(fs1, expected_sip1),
(fs2, expected_sip2),
(fs3, expected_sip3),
(fs5, expected_sip5)]:
for fn, exp_content in expected:
with fs.open(fn) as fp:
if isinstance(exp_content, set):
content = set(fp.read().splitlines())
else:
content = fp.read()
assert content == exp_content