def get_object_entry(args):
'''
Figures out which OE number to use and performs some basic validation.
UNITTEST - use the existing ifs to perform some True/False tests.
'''
if not args.sc:
if args.oe:
if args.oe[:2] != 'oe':
print(
'First two characters must be \'oe\' and last four characters must be four digits'
)
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in list(range(4, 6)):
print(
'First two characters must be \'oe\' and last four characters must be four digits'
)
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print(
'First two characters must be \'oe\' and last four characters must be four digits'
)
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
else:
object_entry = 'not_applicable'
return object_entry
def get_numbers(args):
'''
Figure out the first OE number and how to increment per package.
'''
if args.start_number:
if args.start_number[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.start_number[2:]) != 4:
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.start_number[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.start_number
else:
object_entry = ififuncs.get_object_entry()
object_entry_digits = int(object_entry[2:])
new_object_entry = 'oe' + str(object_entry_digits)
return new_object_entry
def main(args_):
print('\n - Normalise.py started')
args = parse_args(args_)
print(args)
source = args.i
output_folder = args.o
file_list = ififuncs.get_video_files(source)
if args.sip:
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
oe_digits = int(object_entry.replace('oe', ''))
for filename in file_list:
log_name_source = os.path.join(
args.o,
'%s_normalise_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'normalise.py started.')
ififuncs.generate_log(log_name_source,
'Command line arguments: %s' % args)
ififuncs.generate_log(log_name_source, 'EVENT = agentName=%s' % user)
print('\n - Processing: %s' % filename)
ififuncs.generate_log(
log_name_source,
'EVENT = Normalization, status=started, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object to be normalised=%s'
% filename)
output, output_uuid, fmd5, ffv1_logfile = normalise_process(
filename, output_folder)
ififuncs.generate_log(
log_name_source,
'EVENT = Normalization, status=finished, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object normalised into=%s'
% output)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
filename, output_folder, output_uuid, log_name_source)
fmd5_logfile, fmd5ffv1, verdict = verify_losslessness(
output_folder, output, output_uuid, fmd5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation, eventOutcome=%s'
% verdict)
if args.sip:
object_entry_complete = 'oe' + str(oe_digits)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i', output, '-move', '-u', output_uuid, '-user', user, '-oe',
object_entry_complete, '-o', args.o
]
if args.supplement:
supplement_cmd.extend(args.supplement)
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(
sipcreator_cmd)
metadata_dir = os.path.join(
os.path.dirname(os.path.dirname(sipcreator_log)), 'metadata')
shutil.move(fmd5, metadata_dir)
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(fmd5ffv1, metadata_dir)
shutil.move(
ffv1_logfile.replace('\\\\', '\\').replace('\:', ':'),
os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(fmd5)))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(fmd5ffv1)))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(
logs_dir,
os.path.basename(
ffv1_logfile.replace('\\\\', '\\').replace('\:',
':'))))
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(
logs_dir,
os.path.basename(
fmd5_logfile.replace('\\\\', '\\').replace('\:',
':'))))
ififuncs.merge_logs(log_name_source, sipcreator_log,
sipcreator_manifest)
os.remove(dfxml)
os.remove(inputxml)
os.remove(inputtracexml)
oe_digits += 1
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) != 4:
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid
)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(
new_log_textfile,
'EVENT = sipcreator.py started'
)
ififuncs.generate_log(
new_log_textfile,
'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')
)
ififuncs.generate_log(
new_log_textfile,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = agentName=%s' % user
)
ififuncs.generate_log(
new_log_textfile,
uuid_event
)
if args.u is False:
sys.exit()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s'
% object_entry
)
metadata_dir = os.path.join(sip_path, 'metadata')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(
metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir
)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(
logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir))
)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish)
if args.d:
content_title = create_content_title_text(args, sip_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"),
os.path.join('objects', content_title).replace("\\", "/")
)
return new_log_textfile, new_manifest_textfile
def run_loop(args):
'''
This will only process one sequence. Batch processing will come later.
'''
current_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.user:
user = args.user
else:
user = ififuncs.get_user()
object_entry = ififuncs.get_object_entry()
log_name_source = os.path.join(
args.o, '%s_seq2ffv1_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")
)
ififuncs.generate_log(log_name_source, 'seq2ffv1.py started.')
ififuncs.generate_log(
log_name_source,
'eventDetail=seq2ffv1.py %s' % ififuncs.get_script_version('seq2ffv1.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
uuid = ififuncs.create_uuid()
verdicts = []
multi_reeler = False
source_directory = args.i
images = ififuncs.get_image_sequence_files(source_directory)
if images == 'none':
print('no images found in directory - checking for multi-reel sequence')
images = ififuncs.check_multi_reel(source_directory)
multi_reeler = True
if images == 'none':
sys.exit()
# this is checking for a single reeler.
else:
images = [source_directory]
reel_number = 1
objects = []
short_test_reports = []
rawcooked_logfiles = []
for reel in images:
short_test_reports.append(short_test(reel))
for i in short_test_reports:
print((' - 24 frame reversibility test for %s is %s' % (os.path.basename(reel), i)))
if i == 'lossy':
print('It appears that this sequence is not reversible - exiting')
sys.exit()
time.sleep(2)
# check for a/b rolls
if reel[-1] in ['a', 'b']:
reel_number = reel[-2]
ffv1_path, source_abspath, args, log_name_source, normalisation_tool, rawcooked_logfile = make_ffv1(
reel,
args,
log_name_source,
reel_number,
uuid,
multi_reeler
)
objects.append(ffv1_path)
rawcooked_logfiles.append(rawcooked_logfile)
# check for a/b rolls
if not reel[-1] in ['a', 'b']:
reel_number += 1
judgement = package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir)
judgement, sipcreator_log, sipcreator_manifest = judgement
verdicts.append([source_directory, judgement])
for verdict in verdicts:
print(("%-*s : %s" % (50, args.i, verdict[1])))
ififuncs.generate_log(log_name_source, 'seq2ffv1.py finished.')
ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s'
% uuid)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if args.u is False:
sys.exit()
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
if args.d:
content_title = create_content_title_text(args, sip_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
os.path.join('objects', content_title).replace("\\", "/"))
return new_log_textfile, new_manifest_textfile
def main(args_):
'''
Launches the functions that prepare and execute the concatenation.
'''
uuid = ififuncs.create_uuid()
args = parse_args(args_)
print args
log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'concat.py started.')
if args.mov:
container = 'mov'
else:
container = 'mkv'
ififuncs.generate_log(
log_name_source,
'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
source_uuid_check = ''
if os.path.isfile(args.i[0]):
source_uuid = ififuncs.get_source_uuid()
elif os.path.isdir(args.i[0]):
source_uuid_check = ififuncs.check_for_uuid(args)
if source_uuid_check == False:
source_uuid = ififuncs.get_source_uuid()
else: source_uuid = source_uuid_check
ififuncs.generate_log(
log_name_source,
'Relationship, derivation, has source=%s' % source_uuid
)
video_files = args.i
concat_file = ififuncs.get_temp_concat('concat_stuff')
ififuncs.generate_log(
log_name_source,
'concatenation file=%s' % concat_file)
if args.r:
video_files = recursive_file_list(video_files)
video_files = ififuncs.sanitise_filenames(video_files)
for source_files in video_files:
ififuncs.generate_log(
log_name_source,
'source_files = %s' % source_files)
make_chapters(video_files)
ififuncs.concat_textfile(video_files, concat_file)
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid, container)
output_file = os.path.join(args.o, '%s.%s' % (uuid, container))
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
validation_env_dict = ififuncs.set_environment(validation_logfile)
output_bitstream_md5 = subprocess.check_output([
'ffmpeg', '-report',
'-i', output_file,
'-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
], env=validation_env_dict).rstrip()
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
if source_bitstream_md5 == output_bitstream_md5:
print 'process appears to be lossless'
print source_bitstream_md5, output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=pass')
else:
print 'something went wrong - not lossless!'
print source_bitstream_md5,output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=fail')
if args.nochapters != True:
subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
ififuncs.generate_log(
log_name_source,
'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
ififuncs.concat_textfile(video_files, concat_file)
with open(log_name_source, 'r') as concat_log:
concat_lines = concat_log.readlines()
if not args.no_sip:
sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def make_ffv1(
start_number,
source_abspath,
output_dirname,
args,
log_name_source,
user
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if args.sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
# Just perform framemd5 at this stage
if args.rawcooked:
logfile = os.path.join(
temp_dir,
'%s_source_framemd5.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
source_framemd5_env_dict = ififuncs.set_environment(logfile)
source_framemd5_cmd = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-pix_fmt', pix_fmt,
'-f', 'framemd5', source_textfile
]
print(source_abspath)
rawcooked_logfile = os.path.join(
temp_dir, '%s_rawcooked.log' % uuid
)
normalisation_tool = ififuncs.get_rawcooked_version()
files_to_move.append(rawcooked_logfile)
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path]
if args.audio:
rawcooked_cmd.extend([args.audio, '-c:a', 'copy'])
ffv12dpx = (rawcooked_cmd)
else:
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
normalisation_tool = 'FFmpeg'
print(ffv12dpx)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if not args.sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest
def make_ffv1(
start_number,
source_abspath,
output_dirname,
root_filename,
args,
log_name_source
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if not args.no_sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
print ffv12dpx
subprocess.call(ffv12dpx, env=env_dict)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if args.no_sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
sipcreator_log, sipcreator_manifest = sipcreator.main([
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
'Kieran',
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)])
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
return judgement, sipcreator_log, sipcreator_manifest
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if not args.sc:
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
else:
object_entry = 'not_applicable'
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s'
% uuid)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if args.u is False:
sys.exit()
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
if args.d:
content_title = create_content_title_text(sip_path)
new_dcp_path = os.path.join('objects',
content_title).replace("\\", "/")
absolute_dcp_path = os.path.join(sip_path, new_dcp_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
new_dcp_path)
'''
a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT)
b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT)
'''
dcp = DCP(absolute_dcp_path)
clairmeta_version = clairmeta.__version__
dcp_dict = dcp.parse()
# json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': '))
xml_str = dicttoxml.dicttoxml(dcp_dict,
custom_root='ClairmetaProbe',
ids=False,
attr_type=False)
xml_pretty = prettyprint_xml(xml_str)
status, report = dcp.check()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s'
% (status, report, clairmeta_version))
clairmeta_xml = os.path.join(metadata_dir,
'%s_clairmeta.xml' % content_title)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s'
% (clairmeta_xml, clairmeta_version))
with open(clairmeta_xml, 'w') as fo:
fo.write(xml_pretty)
ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile,
'md5')
ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml)
print status
print report
print '\n', user, 'ran this script at %s and it finished at %s' % (
start, finish)
return new_log_textfile, new_manifest_textfile
def main(args_):
''''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
source_folder = args.i
print(args)
oe_dict = {}
user = ififuncs.determine_user(args)
if args.oe:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
oe_digits = int(object_entry.replace('oe', ''))
for folder in sorted(os.listdir(source_folder)):
full_path = os.path.join(source_folder, folder)
if os.path.isdir(full_path):
try:
folder_contents = os.listdir(full_path)
except PermissionError:
continue
object_entry_complete = 'oe' + str(oe_digits)
inputs = []
supplements = []
for files in folder_contents:
if os.path.splitext(files)[1][1:].lower() in args.object_extension_pattern:
inputs.append(os.path.join(full_path, files))
if os.path.splitext(files)[1][1:].lower() in args.supplement_extension_pattern:
supplements.append(os.path.join(full_path, files))
if inputs:
print(' - Object Entry: %s\n - Inputs: %s\n - Supplements: %s\n' % (object_entry_complete, inputs, supplements))
oe_dict[object_entry_complete] = [inputs, supplements]
oe_digits += 1
else:
print('Skipping %s as there are no files in this folder that match the -object_extension_pattern' % full_path)
if args.dryrun:
print('Exiting as you selected -dryrun')
sys.exit()
logs = []
if args.y:
proceed = 'Y'
else:
proceed = ififuncs.ask_yes_no(
'Do you want to proceed?'
)
if proceed == 'Y':
for sips in sorted(oe_dict):
print(oe_dict[sips])
sipcreator_cmd = ['-i',]
for sipcreator_inputs in oe_dict[sips][0]:
sipcreator_cmd.append(sipcreator_inputs)
sipcreator_cmd += ['-supplement']
for sipcreator_supplements in oe_dict[sips][1]:
sipcreator_cmd.append(sipcreator_supplements)
sipcreator_cmd += ['-user', user, '-oe', sips, '-o', args.o]
if args.rename_uuid:
sipcreator_cmd.append('-rename_uuid')
if args.zip:
sipcreator_cmd.append('-zip')
if args.l:
sipcreator_cmd.append('-l')
print(sipcreator_cmd)
sipcreator_log, _ = sipcreator.main(sipcreator_cmd)
logs.append(sipcreator_log)
for i in logs:
if os.path.isfile(i):
print(("%-*s : copyit job was a %s" % (50, os.path.basename(i), analyze_log(i))))
