def test_insert_into_files_raises_if_both_sip_and_transfer_provided(self):
with pytest.raises(Exception) as excinfo:
databaseFunctions.insertIntoFiles("both",
"both_path",
sipUUID="sip",
transferUUID="transfer")
assert "both SIP and transfer UUID" in str(excinfo.value)
def test_insert_into_files_with_sip(self):
path = "%sharedDirectory%/no_such_file"
assert File.objects.filter(currentlocation=path).count() == 0
databaseFunctions.insertIntoFiles(
"uuid", path, sipUUID="0049fa6c-152f-44a0-93b0-c5e856a02292")
assert File.objects.filter(currentlocation=path).count() == 1
def test_insert_into_files_with_original_path(self):
# A filepath set during the extract contents microservice. Note the
# filename contains underscorres from normalization.
file_path = ("%transferDirectory%objects/another_parent_directory/"
"compressed_directory.zip")
# What that path might look like when set correctly in the original
# location field.
original_location = ("%transferDirectory%objects/another parent "
"directory/compressed directory.zip")
# If originalLocation is set, then test that it is set with the right
# value. Check also that we haven't set the current location field
# inaccurately.
databaseFunctions.insertIntoFiles(
fileUUID="e0a1fdc4-605a-4104-bf59-039859ee8238",
filePath=file_path,
sipUUID="0049fa6c-152f-44a0-93b0-c5e856a02292",
originalLocation=original_location)
assert File.objects.filter(originallocation=original_location)\
.count() == 1
assert File.objects.get(
originallocation=original_location)\
.currentlocation != original_location
# If originalLocation is not set (here we use None to be explicit),
# then default to the filePath.
databaseFunctions.insertIntoFiles(
fileUUID="554661f1-b331-452c-a583-0c582ebcb298",
filePath=file_path,
sipUUID="01cf9fb8-bc01-40b4-b830-feb66e912f40",
originalLocation=None)
assert File.objects.filter(
uuid="554661f1-b331-452c-a583-0c582ebcb298")[0]\
.originallocation == file_path
def addFileToSIP(filePathRelativeToSIP, fileUUID, sipUUID, taskUUID, date, sourceType="ingestion", use="original"):
insertIntoFiles(fileUUID, filePathRelativeToSIP, date, sipUUID=sipUUID, use=use)
insertIntoEvents(fileUUID=fileUUID, \
eventType=sourceType, \
eventDateTime=date, \
eventDetail="", \
eventOutcome="", \
eventOutcomeDetailNote="")
def addFileToSIP(filePathRelativeToSIP, fileUUID, sipUUID, taskUUID, date, sourceType="ingestion", use="original"):
insertIntoFiles(fileUUID, filePathRelativeToSIP, date, sipUUID=sipUUID, use=use)
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=taskUUID, \
eventType=sourceType, \
eventDateTime=date, \
eventDetail="", \
eventOutcome="", \
eventOutcomeDetailNote="")
def addFileToTransfer(filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType="ingestion", eventDetail="", use="original"):
#print filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType, eventDetail, use
insertIntoFiles(fileUUID, filePathRelativeToSIP, date, transferUUID=transferUUID, use=use)
insertIntoEvents(fileUUID=fileUUID, \
eventType=sourceType, \
eventDateTime=date, \
eventDetail=eventDetail, \
eventOutcome="", \
eventOutcomeDetailNote="")
addAccessionEvent(fileUUID, transferUUID, date)
def addFileToTransfer(filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType="ingestion", eventDetail="", use="original"):
#print filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType, eventDetail, use
insertIntoFiles(fileUUID, filePathRelativeToSIP, date, transferUUID=transferUUID, use=use)
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=taskUUID, \
eventType=sourceType, \
eventDateTime=date, \
eventDetail=eventDetail, \
eventOutcome="", \
eventOutcomeDetailNote="")
addAccessionEvent(fileUUID, transferUUID, date)
def addFileToTransfer(filePathRelativeToSIP, fileUUID, transferUUID, taskUUID,
date, sourceType="ingestion", eventDetail="",
use="original", originalLocation=None):
if not originalLocation:
originalLocation = filePathRelativeToSIP
insertIntoFiles(
fileUUID, filePathRelativeToSIP, date,
transferUUID=transferUUID, use=use, originalLocation=originalLocation)
insertIntoEvents(fileUUID=fileUUID,
eventType=sourceType,
eventDateTime=date,
eventDetail=eventDetail,
eventOutcome="",
eventOutcomeDetailNote="")
addAccessionEvent(fileUUID, transferUUID, date)
def compress_aip(job, compression, compression_level, sip_directory, sip_name,
sip_uuid):
""" Compresses AIP according to compression algorithm and level.
compression = AIP compression algorithm, format: <program>-<algorithm>, eg. 7z-lzma, pbzip2-
compression_level = AIP compression level, integer between 1 and 9 inclusive
sip_directory = Absolute path to the directory where the SIP is
sip_name = User-provided name of the SIP
sip_uuid = SIP UUID
Example inputs:
compressAIP.py
7z-lzma
5
/var/archivematica/sharedDirectory/watchedDirectories/workFlowDecisions/compressionAIPDecisions/ep-d87d5845-bd07-4200-b1a4-928e0cb6e1e4/
ep
d87d5845-bd07-4200-b1a4-928e0cb6e1e4
"""
try:
program, compression_algorithm = compression.split('-')
except ValueError:
msg = "Invalid program-compression algorithm: {}".format(compression)
job.pyprint(msg, file=sys.stderr)
return 255
archive_path = '{name}-{uuid}'.format(name=sip_name, uuid=sip_uuid)
uncompressed_location = sip_directory + archive_path
# Even though no actual compression is taking place,
# the location still needs to be set in the unit to ensure that the
# %AIPFilename% variable is set appropriately.
# Setting it to an empty string ensures the common
# "%SIPDirectory%%AIPFilename%" pattern still points at the right thing.
if program == 'None':
update_unit(sip_uuid, uncompressed_location)
return 0
job.pyprint("Compressing {} with {}, algorithm {}, level {}".format(
uncompressed_location, program, compression_algorithm,
compression_level))
if program == '7z':
compressed_location = uncompressed_location + ".7z"
command = '/usr/bin/7z a -bd -t7z -y -m0={algorithm} -mx={level} -mta=on -mtc=on -mtm=on -mmt=on "{compressed_location}" "{uncompressed_location}"'.format(
algorithm=compression_algorithm,
level=compression_level,
uncompressed_location=uncompressed_location,
compressed_location=compressed_location)
tool_info_command = (
'echo program="7z"\; '
'algorithm="{}"\; '
'version="`7z | grep Version`"'.format(compression_algorithm))
elif program == 'pbzip2':
compressed_location = uncompressed_location + ".tar.bz2"
command = '/bin/tar -c --directory "{sip_directory}" "{archive_path}" | /usr/bin/pbzip2 --compress -{level} > "{compressed_location}"'.format(
sip_directory=sip_directory,
archive_path=archive_path,
level=compression_level,
compressed_location=compressed_location)
tool_info_command = (
'echo program="pbzip2"\; '
'algorithm="{}"\; '
'version="$((pbzip2 -V) 2>&1)"'.format(compression_algorithm))
else:
msg = "Program {} not recognized, exiting script prematurely.".format(
program)
job.pyprint(msg, file=sys.stderr)
return 255
job.pyprint('Executing command:', command)
exit_code, std_out, std_err = executeOrRun("bashScript",
command,
printing=True,
capture_output=True)
job.write_output(std_out)
job.write_error(std_err)
# Add new AIP File
file_uuid = sip_uuid
databaseFunctions.insertIntoFiles(
fileUUID=file_uuid,
filePath=compressed_location.replace(sip_directory, '%SIPDirectory%',
1),
sipUUID=sip_uuid,
use='aip',
)
# Add compression event
job.pyprint('Tool info command:', tool_info_command)
_, tool_info, tool_info_err = executeOrRun("bashScript",
tool_info_command,
printing=True,
capture_output=True)
job.write_output(tool_info)
job.write_error(tool_info_err)
tool_output = 'Standard Output="{}"; Standard Error="{}"'.format(
std_out, std_err)
databaseFunctions.insertIntoEvents(
eventType='compression',
eventDetail=tool_info,
eventOutcomeDetailNote=tool_output,
fileUUID=file_uuid,
)
update_unit(sip_uuid, compressed_location)
return exit_code
def test_insert_into_files_raises_if_no_sip_or_transfer_provided(self):
with pytest.raises(Exception) as excinfo:
databaseFunctions.insertIntoFiles("no_sip", "no_sip_path")
assert "neither defined" in str(excinfo.value)
