def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
clairmeta_version = clairmeta.__version__
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print(args)
user = ififuncs.determine_user(args)
object_entry = get_object_entry(args)
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
uuid, uuid_event = determine_uuid(args, sip_path)
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
if args.d:
content_title = create_content_title_text(sip_path, args)
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
if args.accession:
accession_number = ififuncs.get_accession_number()
reference_number = ififuncs.get_reference_number()
parent = ififuncs.ask_question(
'What is the parent record? eg MV 1234. Enter n/a if this is a born digital acquisition with no parent.'
)
donor = ififuncs.ask_question(
'Who is the source of acquisition, as appears on the donor agreement? This will not affect Reproductions.'
)
depositor_reference = ififuncs.ask_question(
'What is the donor/depositor number? This will not affect Reproductions.'
)
acquisition_type = ififuncs.get_acquisition_type('')
donation_date = ififuncs.ask_question(
'When was the donation date in DD/MM/YYYY format? Eg. 31/12/1999 - Unfortunately this is NOT using ISO 8601.'
)
if args.zip:
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
inputs[0], args.o, uuid, new_log_textfile)
if args.manifest:
shutil.copy(
args.manifest,
args.manifest.replace('_manifest.md5', '_manifest-md5.txt'))
source_manifest = args.manifest.replace('_manifest.md5',
'_manifest-md5.txt')
else:
source_manifest = os.path.join(
args.o,
os.path.basename(args.i[0]) + '_manifest-md5.txt')
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.hashlib_manifest(args.i[0], source_manifest,
os.path.dirname(args.i[0]))
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
% inputs[0])
makezip_judgement, zip_file = makezip.main([
'-i', inputs[0], '-o',
os.path.join(sip_path, 'objects'), '-basename', uuid + '.zip'
])
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, eventDetail=Source object packed into=%s'
% zip_file)
if makezip_judgement is None:
judgement = 'lossless'
else:
judgement = makezip_judgement
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
else:
log_names = move_files(inputs, sip_path, args, user)
ififuncs.get_technical_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
if args.zip:
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=started, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
ififuncs.manifest_update(new_manifest_textfile, zip_file)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=finished, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.zip:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', [inputxml, inputtracexml, dfxml, source_manifest], '-user',
user, '-new_folder', supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
if 'log_names' in locals():
log_report(log_names)
finish = datetime.datetime.now()
print('\n- %s ran this script at %s and it finished at %s' %
(user, start, finish))
if args.d:
process_dcp(sip_path, content_title, args, new_manifest_textfile,
new_log_textfile, metadata_dir, clairmeta_version)
if args.accession:
register = accession.make_register()
filmographic_dict = ififuncs.extract_metadata(args.csv)[0]
for filmographic_record in filmographic_dict:
if filmographic_record['Reference Number'].lower(
) == reference_number.lower():
if filmographic_record['Title'] == '':
title = filmographic_record[
'TitleSeries'] + '; ' + filmographic_record['EpisodeNo']
else:
title = filmographic_record['Title']
oe_register = make_oe_register()
ififuncs.append_csv(
oe_register,
(object_entry.upper()[:2] + '-' + object_entry[2:], donation_date,
'1', '', title, donor, acquisition_type[1], accession_number,
'Representation of %s|Reproduction of %s' %
(reference_number, parent), ''))
accession_cmd = [
os.path.dirname(sip_path), '-user', user, '-f', '-number',
accession_number, '-reference', reference_number, '-register',
register, '-csv', args.csv, '-pbcore'
]
if not parent.lower() == 'n/a':
accession_cmd.extend(['-parent', parent])
accession_cmd.extend(['-donor', donor])
accession_cmd.extend(['-depositor_reference', depositor_reference])
accession_cmd.extend(['-acquisition_type', acquisition_type[2]])
accession_cmd.extend(['-donation_date', donation_date])
print(accession_cmd)
accession.main(accession_cmd)
return new_log_textfile, new_manifest_textfile
def package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir):
'''
Package the MKV using sipcreator.py
'''
sip_dir = os.path.join(
args.o, os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(source_abspath, args.o, uuid, log_name_source)
source_manifest = os.path.join(
args.o,
os.path.basename(args.i) + '_manifest-md5.txt'
)
ififuncs.generate_log(
log_name_source,
'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files'
)
if multi_reeler:
ififuncs.hashlib_manifest(args.i, source_manifest, args.i)
else:
ififuncs.hashlib_manifest(args.i, source_manifest, os.path.dirname(args.i))
ififuncs.generate_log(
log_name_source,
'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files'
)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibility of %s back to its original form, followed by checksum verification using %s ' % (normalisation_tool, objects, source_manifest)
)
if args.reversibility_dir:
reversibility_dir = args.reversibility_dir
else:
reversibility_dir = args.o
judgement = reversibility_verification(objects, source_manifest, reversibility_dir)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibilty of %s back to its original form, followed by checksum verification using %s , eventOutcome=%s' % (normalisation_tool, objects, source_manifest, judgement)
)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml, source_manifest]
if args.supplement:
supplement_cmd.extend(args.supplement)
sipcreator_cmd = [
'-i',
]
for i in objects:
sipcreator_cmd.append(i)
sipcreator_cmd += [
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', args.o
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
for files in os.listdir(logs_dir):
if files.endswith('.md5'):
deletefiles.main(['-i', os.path.join(logs_dir, files), '-uuid_path', sip_dir, '-user', user])
for rawcooked_logfile in rawcooked_logfiles:
rawcooked_logfile = rawcooked_logfile.replace('\'', '')
shutil.move(rawcooked_logfile, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(rawcooked_logfile))
)
metadata_dir = os.path.join(sip_dir, 'metadata')
os.chdir(current_dir)
shutil.copy(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'film_scan_aip_documentation.txt'), metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, 'film_scan_aip_documentation.txt')
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest
def main(args_):
print('\n - Normalise.py started')
args = parse_args(args_)
print(args)
source = args.i
output_folder = args.o
file_list = ififuncs.get_video_files(source)
if args.sip:
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
oe_digits = int(object_entry.replace('oe', ''))
for filename in file_list:
log_name_source = os.path.join(
args.o,
'%s_normalise_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'normalise.py started.')
ififuncs.generate_log(log_name_source,
'Command line arguments: %s' % args)
ififuncs.generate_log(log_name_source, 'EVENT = agentName=%s' % user)
print('\n - Processing: %s' % filename)
ififuncs.generate_log(
log_name_source,
'EVENT = Normalization, status=started, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object to be normalised=%s'
% filename)
output, output_uuid, fmd5, ffv1_logfile = normalise_process(
filename, output_folder)
ififuncs.generate_log(
log_name_source,
'EVENT = Normalization, status=finished, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object normalised into=%s'
% output)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
filename, output_folder, output_uuid, log_name_source)
fmd5_logfile, fmd5ffv1, verdict = verify_losslessness(
output_folder, output, output_uuid, fmd5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation, eventOutcome=%s'
% verdict)
if args.sip:
object_entry_complete = 'oe' + str(oe_digits)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i', output, '-move', '-u', output_uuid, '-user', user, '-oe',
object_entry_complete, '-o', args.o
]
if args.supplement:
supplement_cmd.extend(args.supplement)
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(
sipcreator_cmd)
metadata_dir = os.path.join(
os.path.dirname(os.path.dirname(sipcreator_log)), 'metadata')
shutil.move(fmd5, metadata_dir)
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(fmd5ffv1, metadata_dir)
shutil.move(
ffv1_logfile.replace('\\\\', '\\').replace('\:', ':'),
os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(fmd5)))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(fmd5ffv1)))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(
logs_dir,
os.path.basename(
ffv1_logfile.replace('\\\\', '\\').replace('\:',
':'))))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(
logs_dir,
os.path.basename(
fmd5_logfile.replace('\\\\', '\\').replace('\:',
':'))))
ififuncs.merge_logs(log_name_source, sipcreator_log,
sipcreator_manifest)
os.remove(dfxml)
os.remove(inputxml)
os.remove(inputtracexml)
oe_digits += 1
def make_ffv1(
start_number,
source_abspath,
output_dirname,
args,
log_name_source,
user
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if args.sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
# Just perform framemd5 at this stage
if args.rawcooked:
logfile = os.path.join(
temp_dir,
'%s_source_framemd5.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
source_framemd5_env_dict = ififuncs.set_environment(logfile)
source_framemd5_cmd = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-pix_fmt', pix_fmt,
'-f', 'framemd5', source_textfile
]
print(source_abspath)
rawcooked_logfile = os.path.join(
temp_dir, '%s_rawcooked.log' % uuid
)
normalisation_tool = ififuncs.get_rawcooked_version()
files_to_move.append(rawcooked_logfile)
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path]
if args.audio:
rawcooked_cmd.extend([args.audio, '-c:a', 'copy'])
ffv12dpx = (rawcooked_cmd)
else:
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
normalisation_tool = 'FFmpeg'
print(ffv12dpx)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if not args.sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest