def short_test(images, args):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
maybe all that needs to happen is that 24 frames are copied to a temp
location, then the functions run, and use ififuncs to check the sums
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
for image in images[:24]:
full_path = os.path.join(args.i, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
ififuncs.diff_textfiles(converted_manifest, restored_manifest)
def make_ffv1(reel, args, log_name_source, reel_number, uuid, multi_reeler):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
output_dirname = args.o
if multi_reeler:
mkv_basename = uuid + '_reel%s.mkv' % str(reel_number)
else:
mkv_basename = uuid + '.mkv'
ffv1_path = os.path.join(output_dirname, mkv_basename)
rawcooked_logfile = os.path.join(args.o, '%s_rawcooked.log' % mkv_basename)
normalisation_tool = ififuncs.get_rawcooked_version()
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
rawcooked_cmd = [
'rawcooked', reel, '--check', 'full', '-c:a', 'copy', '-o', ffv1_path
]
ffv12dpx = (rawcooked_cmd)
print(ffv12dpx)
if args.zip:
uuid = ififuncs.create_uuid()
# ugly hack until i recfactor. this is the zip_path, not ffv1_path
ffv1_path = os.path.join(output_dirname, uuid + '.zip')
ififuncs.generate_log(
log_name_source,
'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
% reel)
makezip_judgement = makezip.main([
'-i', reel, '-o', output_dirname, '-basename',
os.path.basename(ffv1_path)
])[0]
ififuncs.generate_log(
log_name_source,
'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, Source object packed into=%s'
% ffv1_path)
if makezip_judgement is None:
judgement = 'lossless'
else:
judgement = makezip_judgement
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
if not args.zip:
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool)
return ffv1_path, reel, args, log_name_source, normalisation_tool, rawcooked_logfile
def normalise_process(filename, output_folder):
'''
Begins the actual normalisation process using FFmpeg
'''
output_uuid = ififuncs.create_uuid()
print(' - The following UUID has been generated: %s' % output_uuid)
output = "%s/%s.mkv" % (
output_folder, output_uuid
)
print(' - The normalised file will have this filename: %s' % output)
fmd5 = "%s/%s_source.framemd5" % (
output_folder, os.path.basename(filename)
)
print(' - Framemd5s for each frame of your input file will be stored in: %s' % fmd5)
ffv1_logfile = os.path.join(output_folder, '%s_normalise.log' % output_uuid)
print(' - The FFmpeg logfile for the transcode will be stored in: %s' % ffv1_logfile)
print(' - FFmpeg will begin normalisation now.')
ffv1_env_dict = ififuncs.set_environment(ffv1_logfile)
ffv1_command = [
'ffmpeg',
'-i', filename,
'-c:v', 'ffv1', # Use FFv1 codec
'-g', '1', # Use intra-frame only aka ALL-I aka GOP=1
'-level', '3', # Use Version 3 of FFv1
'-c:a', 'copy', # Copy and paste audio bitsream with no transcoding
'-map', '0',
'-dn',
'-report',
'-slicecrc', '1',
'-slices', '16',
]
if ififuncs.check_for_fcp(filename) is True:
print(' - A 720/576 file with no Pixel Aspect Ratio and scan type metadata has been detected.')
ffv1_command += [
'-vf',
'setfield=tff, setdar=4/3'
]
print(' - -vf setfield=tff, setdar=4/3 will be added to the FFmpeg command.')
ffprobe_dict = ififuncs.get_ffprobe_dict(filename)
# let's stipulate the colour metadata if not present for SD PAL material.
if not ififuncs.get_colour_metadata(ffprobe_dict):
ffv1_command += ['-color_primaries', 'bt470bg', '-color_trc', 'bt709', '-colorspace', 'bt470bg' ]
ffv1_command += [
output,
'-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print(ffv1_command)
subprocess.call(ffv1_command, env=ffv1_env_dict)
return output, output_uuid, fmd5, ffv1_logfile
def short_test(images):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
# Only analyse the first 24 frames.
for image in sorted(os.listdir(images))[:24]:
full_path = os.path.join(images, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-c:a', 'copy', '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, restored_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp reversibility directory %s' % rawcooked_dir))
try:
shutil.rmtree(rawcooked_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp FFV1/MKV %s' % mkv_file))
os.remove(mkv_file)
return judgement
def make_folder_path(path, args, object_entry):
'''
Generates objects/logs/metadata/UUID folder structure in output.
Returns the path.
'''
if not args.u:
representation_uuid = ififuncs.create_uuid()
else:
representation_uuid = args.u
oe_path = os.path.join(path, object_entry)
path = os.path.join(oe_path, representation_uuid)
print path
ififuncs.make_folder_structure(path)
return path
def make_folder_path(path, args, object_entry):
'''
Generates objects/logs/metadata/UUID folder structure in output.
Returns the path.
'''
if not args.u:
representation_uuid = ififuncs.create_uuid()
else:
representation_uuid = args.u
oe_path = os.path.join(path, object_entry)
path = os.path.join(oe_path, representation_uuid)
print path
ififuncs.make_folder_structure(path)
return path
def make_folder_path(path, args, object_entry):
'''
Generates objects/logs/metadata/UUID folder structure in output directory.
Asks user for UUID if it's not supplied in an arg.
Adds a workaround for special collections workflows.
Returns the path.
UNITTEST - does path exist
'''
if not args.u:
representation_uuid = ififuncs.create_uuid()
else:
representation_uuid = args.u
if args.sc:
oe_path = args.o
else:
oe_path = os.path.join(path, object_entry)
path = os.path.join(oe_path, representation_uuid)
ififuncs.make_folder_structure(path)
return path
def make_folders(parent, args):
'''
Actually make the folders!
'''
output = args.o
parent_folder = os.path.join(output, parent)
os.makedirs(parent_folder)
uuid_folder = os.path.join(parent_folder, ififuncs.create_uuid())
logs_dir = os.path.join(uuid_folder, 'logs')
objects_dir = os.path.join(uuid_folder, 'objects')
metadata_dir = os.path.join(uuid_folder, 'metadata')
os.makedirs(logs_dir)
os.makedirs(objects_dir)
os.makedirs(metadata_dir)
for folder in [logs_dir, objects_dir, metadata_dir]:
print folder
os.chdir(folder)
os.makedirs('image')
os.makedirs('audio')
def reversibility_verification(objects, source_manifest, reversibility_dir):
'''
Restore the MKV back to DPX, create a checksum, and compare to source DPX
checksums.
Return a value of lossy or lossless.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(reversibility_dir, temp_uuid)
os.makedirs(temp_dir)
for ffv1_mkv in objects:
subprocess.call(['rawcooked', ffv1_mkv, '-o', temp_dir])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, source_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Unable to delete temp directory, sorry!')
return judgement
def run_loop(args):
'''
This will only process one sequence. Batch processing will come later.
'''
current_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
if args.user:
user = args.user
else:
user = ififuncs.get_user()
object_entry = ififuncs.get_object_entry()
log_name_source = os.path.join(
args.o, '%s_seq2ffv1_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")
)
ififuncs.generate_log(log_name_source, 'seq2ffv1.py started.')
ififuncs.generate_log(
log_name_source,
'eventDetail=seq2ffv1.py %s' % ififuncs.get_script_version('seq2ffv1.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
uuid = ififuncs.create_uuid()
verdicts = []
multi_reeler = False
source_directory = args.i
images = ififuncs.get_image_sequence_files(source_directory)
if images == 'none':
print('no images found in directory - checking for multi-reel sequence')
images = ififuncs.check_multi_reel(source_directory)
multi_reeler = True
if images == 'none':
sys.exit()
# this is checking for a single reeler.
else:
images = [source_directory]
reel_number = 1
objects = []
short_test_reports = []
rawcooked_logfiles = []
for reel in images:
short_test_reports.append(short_test(reel))
for i in short_test_reports:
print((' - 24 frame reversibility test for %s is %s' % (os.path.basename(reel), i)))
if i == 'lossy':
print('It appears that this sequence is not reversible - exiting')
sys.exit()
time.sleep(2)
# check for a/b rolls
if reel[-1] in ['a', 'b']:
reel_number = reel[-2]
ffv1_path, source_abspath, args, log_name_source, normalisation_tool, rawcooked_logfile = make_ffv1(
reel,
args,
log_name_source,
reel_number,
uuid,
multi_reeler
)
objects.append(ffv1_path)
rawcooked_logfiles.append(rawcooked_logfile)
# check for a/b rolls
if not reel[-1] in ['a', 'b']:
reel_number += 1
judgement = package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir)
judgement, sipcreator_log, sipcreator_manifest = judgement
verdicts.append([source_directory, judgement])
for verdict in verdicts:
print(("%-*s : %s" % (50, args.i, verdict[1])))
ififuncs.generate_log(log_name_source, 'seq2ffv1.py finished.')
ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main():
'''
Prints a new UUID to the terminal
'''
new_uuid = create_uuid()
print new_uuid
def main(args_):
'''
Launches the functions that prepare and execute the concatenation.
'''
uuid = ififuncs.create_uuid()
args = parse_args(args_)
print args
log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'concat.py started.')
if args.mov:
container = 'mov'
else:
container = 'mkv'
ififuncs.generate_log(
log_name_source,
'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
source_uuid_check = ''
if os.path.isfile(args.i[0]):
source_uuid = ififuncs.get_source_uuid()
elif os.path.isdir(args.i[0]):
source_uuid_check = ififuncs.check_for_uuid(args)
if source_uuid_check == False:
source_uuid = ififuncs.get_source_uuid()
else: source_uuid = source_uuid_check
ififuncs.generate_log(
log_name_source,
'Relationship, derivation, has source=%s' % source_uuid
)
video_files = args.i
concat_file = ififuncs.get_temp_concat('concat_stuff')
ififuncs.generate_log(
log_name_source,
'concatenation file=%s' % concat_file)
if args.r:
video_files = recursive_file_list(video_files)
video_files = ififuncs.sanitise_filenames(video_files)
for source_files in video_files:
ififuncs.generate_log(
log_name_source,
'source_files = %s' % source_files)
make_chapters(video_files)
ififuncs.concat_textfile(video_files, concat_file)
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid, container)
output_file = os.path.join(args.o, '%s.%s' % (uuid, container))
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
validation_env_dict = ififuncs.set_environment(validation_logfile)
output_bitstream_md5 = subprocess.check_output([
'ffmpeg', '-report',
'-i', output_file,
'-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
], env=validation_env_dict).rstrip()
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
if source_bitstream_md5 == output_bitstream_md5:
print 'process appears to be lossless'
print source_bitstream_md5, output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=pass')
else:
print 'something went wrong - not lossless!'
print source_bitstream_md5,output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=fail')
if args.nochapters != True:
subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
ififuncs.generate_log(
log_name_source,
'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
ififuncs.concat_textfile(video_files, concat_file)
with open(log_name_source, 'r') as concat_log:
concat_lines = concat_log.readlines()
if not args.no_sip:
sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main(args_):
'''
Retrospectively updates older FFV1/DV packages in order to meet our current
packaging requirements. This should allow accession.py and makepbcore.py to run as
expected. This script should work on files created by:
makeffv1.py
dvsip.py
loopline.py
'''
args = parse_args(args_)
user = ififuncs.get_user()
new_object_entry = get_numbers(args)
for root, _, filenames in os.walk(args.input):
if os.path.basename(root)[:2] == 'oe':
if len(os.path.basename(root)[2:]) == 4:
log_dir = os.path.join(root, 'logs')
for files in os.listdir(log_dir):
if '.mov_log.log' in files:
log = os.path.join(log_dir, files)
old_oe_path = root
old_oe = os.path.basename(root)
manifest = os.path.join(os.path.dirname(root),
old_oe + '_manifest.md5')
uuid = ififuncs.create_uuid()
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
ififuncs.generate_log(log,
'EVENT = loopline_repackage.py started')
ififuncs.generate_log(
log, 'eventDetail=loopline_repackage.py %s' %
ififuncs.get_script_version('loopline_repackage.py'))
ififuncs.generate_log(log, 'Command line arguments: %s' % args)
ififuncs.generate_log(log, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(log, uuid_event)
ififuncs.generate_log(
log, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' %
new_object_entry)
old_uuid_path = os.path.join(os.path.dirname(root), uuid)
new_oe_path, new_uuid_path = move_files(
root, new_object_entry, old_oe_path, old_uuid_path, uuid)
updated_lines = update_manifest(manifest, old_oe, uuid)
new_manifest = os.path.join(new_oe_path,
uuid) + '_manifest.md5'
shutil.move(manifest, new_manifest)
with open(new_manifest, 'w') as fo:
for lines in updated_lines:
fo.write(lines)
new_logs_path = os.path.join(new_uuid_path, 'logs')
for files in os.listdir(new_logs_path):
if '.mov_log.log' in files:
log = os.path.join(new_logs_path, files)
logname = rename_files(new_uuid_path, old_oe, uuid,
new_manifest, log)
ififuncs.generate_log(
logname, 'EVENT = loopline_repackage.py finished')
ififuncs.checksum_replace(new_manifest, logname, 'md5')
oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1
new_object_entry = 'oe' + str(oe_digits)
def make_ffv1(
start_number,
source_abspath,
output_dirname,
args,
log_name_source,
user
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if args.sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
# Just perform framemd5 at this stage
if args.rawcooked:
logfile = os.path.join(
temp_dir,
'%s_source_framemd5.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
source_framemd5_env_dict = ififuncs.set_environment(logfile)
source_framemd5_cmd = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-pix_fmt', pix_fmt,
'-f', 'framemd5', source_textfile
]
print(source_abspath)
rawcooked_logfile = os.path.join(
temp_dir, '%s_rawcooked.log' % uuid
)
normalisation_tool = ififuncs.get_rawcooked_version()
files_to_move.append(rawcooked_logfile)
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path]
if args.audio:
rawcooked_cmd.extend([args.audio, '-c:a', 'copy'])
ffv12dpx = (rawcooked_cmd)
else:
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
normalisation_tool = 'FFmpeg'
print(ffv12dpx)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if not args.sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest
def main(args_):
'''
Retrospectively updates older FFV1/DV packages in order to meet our current
packaging requirements. This should allow accession.py and makepbcore.py to run as
expected. This script should work on files created by:
makeffv1.py
dvsip.py
loopline.py
'''
args = parse_args(args_)
user = ififuncs.get_user()
new_object_entry = get_numbers(args)
filmographic_csv = args.filmographic
technical_csv = args.technical
filmographic_oe_list = []
filmo_csv_extraction = ififuncs.extract_metadata(filmographic_csv)
tech_csv_extraction = ififuncs.extract_metadata(technical_csv)
register = make_register()
for line_item in filmo_csv_extraction[0]:
dictionary = {}
oe_number = line_item['Object Entry'].lower()
dictionary['title'] = line_item['Title']
if dictionary['title'] == '':
dictionary['title'] = '%s - %s' % (line_item['TitleSeries'],
line_item['EpisodeNo'])
dictionary['uppercase_dashed_oe'] = oe_number.upper()
for tech_record in tech_csv_extraction[0]:
if tech_record['Reference Number'] == dictionary[
'uppercase_dashed_oe']:
dictionary['source_accession_number'] = tech_record[
'Accession Number']
dictionary['filmographic_reference_number'] = tech_record[
'new_ref']
# this transforms OE-#### to oe####
dictionary['old_oe'] = oe_number[:2] + oe_number[3:]
filmographic_oe_list.append(dictionary)
for oe_package in filmographic_oe_list:
for root, _, filenames in os.walk(args.input):
if os.path.basename(root) == oe_package['old_oe']:
old_oe_path = root
old_oe = os.path.basename(root)
log_dir = os.path.join(root, 'logs')
for files in os.listdir(log_dir):
if '.mov_log.log' in files:
log = os.path.join(log_dir, files)
manifest = os.path.join(os.path.dirname(root),
old_oe + '_manifest.md5')
uuid = ififuncs.create_uuid()
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
ififuncs.generate_log(log,
'EVENT = loopline_repackage.py started')
ififuncs.generate_log(
log, 'eventDetail=loopline_repackage.py %s' %
ififuncs.get_script_version('loopline_repackage.py'))
ififuncs.generate_log(log, 'Command line arguments: %s' % args)
ififuncs.generate_log(log, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(log, uuid_event)
ififuncs.generate_log(
log, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' %
new_object_entry)
ififuncs.generate_log(
log, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=Filmographic reference number , value=%s'
% oe_package['filmographic_reference_number'])
oe_package['new_object_entry'] = new_object_entry
print('Transforming %s into %s' %
(oe_package['old_oe'], oe_package['new_object_entry']))
ififuncs.generate_log(
log, 'Relationship, derivation, has source=%s' %
oe_package['source_accession_number'])
old_uuid_path = os.path.join(os.path.dirname(root), uuid)
new_oe_path, new_uuid_path = move_files(
root, new_object_entry, old_oe_path, old_uuid_path, uuid)
updated_lines = update_manifest(manifest, old_oe, uuid)
new_manifest = os.path.join(new_oe_path,
uuid) + '_manifest.md5'
shutil.move(manifest, new_manifest)
with open(new_manifest, 'w') as fo:
for lines in updated_lines:
fo.write(lines)
new_logs_path = os.path.join(new_uuid_path, 'logs')
for files in os.listdir(new_logs_path):
if '.mov_log.log' in files:
log = os.path.join(new_logs_path, files)
logname = rename_files(new_uuid_path, old_oe, uuid,
new_manifest, log)
date_modified, extension = get_date_modified(new_uuid_path)
# This normally would be bad practise, but this project only has two formats. MOV/DV and FFv1/MKV
if extension == '.mkv':
av_format = 'FFV1/PCM/Matroska'
elif extension == '.mov':
av_format = 'DV/PCM/QuickTime'
provenance_string = 'Reproduction of %s' % oe_package[
'source_accession_number']
ififuncs.append_csv(
register,
(oe_package['new_object_entry'].upper()[:2] + '-' +
oe_package['new_object_entry'][2:], date_modified, '1',
av_format, oe_package['title'], 'contact_name',
'Reproduction', '', provenance_string, '', ''))
ififuncs.generate_log(
logname, 'EVENT = loopline_repackage.py finished')
ififuncs.checksum_replace(new_manifest, logname, 'md5')
oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1
new_object_entry = 'oe' + str(oe_digits)
def make_ffv1(
start_number,
source_abspath,
output_dirname,
root_filename,
args,
log_name_source
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if not args.no_sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
print ffv12dpx
subprocess.call(ffv12dpx, env=env_dict)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if args.no_sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
sipcreator_log, sipcreator_manifest = sipcreator.main([
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
'Kieran',
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)])
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
return judgement, sipcreator_log, sipcreator_manifest
