def short_test(images, args): ''' Perform a test on the first 24 frames that will encode via Rawcooked, then decode back to the original form, and the whole file checksums of the original 24 frames and the restored 24 frames are compared. maybe all that needs to happen is that 24 frames are copied to a temp location, then the functions run, and use ififuncs to check the sums ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid) os.makedirs(temp_dir) for image in images[:24]: full_path = os.path.join(args.i, image) shutil.copy(full_path, temp_dir) mkv_uuid = ififuncs.create_uuid() mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv') subprocess.call(['rawcooked', temp_dir, '-o', mkv_file]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) subprocess.call(['rawcooked', mkv_file]) rawcooked_dir = mkv_file + '.RAWcooked' restored_dir = os.path.join(rawcooked_dir, temp_uuid) restored_manifest = os.path.join(restored_dir, '456.md5') ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir) ififuncs.diff_textfiles(converted_manifest, restored_manifest)
def make_ffv1(reel, args, log_name_source, reel_number, uuid, multi_reeler): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' output_dirname = args.o if multi_reeler: mkv_basename = uuid + '_reel%s.mkv' % str(reel_number) else: mkv_basename = uuid + '.mkv' ffv1_path = os.path.join(output_dirname, mkv_basename) rawcooked_logfile = os.path.join(args.o, '%s_rawcooked.log' % mkv_basename) normalisation_tool = ififuncs.get_rawcooked_version() rawcooked_logfile = "\'" + rawcooked_logfile + "\'" env_dict = ififuncs.set_environment(rawcooked_logfile) rawcooked_cmd = [ 'rawcooked', reel, '--check', 'full', '-c:a', 'copy', '-o', ffv1_path ] ffv12dpx = (rawcooked_cmd) print(ffv12dpx) if args.zip: uuid = ififuncs.create_uuid() # ugly hack until i recfactor. this is the zip_path, not ffv1_path ffv1_path = os.path.join(output_dirname, uuid + '.zip') ififuncs.generate_log( log_name_source, 'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s' % reel) makezip_judgement = makezip.main([ '-i', reel, '-o', output_dirname, '-basename', os.path.basename(ffv1_path) ])[0] ififuncs.generate_log( log_name_source, 'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, Source object packed into=%s' % ffv1_path) if makezip_judgement is None: judgement = 'lossless' else: judgement = makezip_judgement ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s' % judgement) if not args.zip: ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool) subprocess.call(ffv12dpx, env=env_dict) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool) return ffv1_path, reel, args, log_name_source, normalisation_tool, rawcooked_logfile
def normalise_process(filename, output_folder): ''' Begins the actual normalisation process using FFmpeg ''' output_uuid = ififuncs.create_uuid() print(' - The following UUID has been generated: %s' % output_uuid) output = "%s/%s.mkv" % ( output_folder, output_uuid ) print(' - The normalised file will have this filename: %s' % output) fmd5 = "%s/%s_source.framemd5" % ( output_folder, os.path.basename(filename) ) print(' - Framemd5s for each frame of your input file will be stored in: %s' % fmd5) ffv1_logfile = os.path.join(output_folder, '%s_normalise.log' % output_uuid) print(' - The FFmpeg logfile for the transcode will be stored in: %s' % ffv1_logfile) print(' - FFmpeg will begin normalisation now.') ffv1_env_dict = ififuncs.set_environment(ffv1_logfile) ffv1_command = [ 'ffmpeg', '-i', filename, '-c:v', 'ffv1', # Use FFv1 codec '-g', '1', # Use intra-frame only aka ALL-I aka GOP=1 '-level', '3', # Use Version 3 of FFv1 '-c:a', 'copy', # Copy and paste audio bitsream with no transcoding '-map', '0', '-dn', '-report', '-slicecrc', '1', '-slices', '16', ] if ififuncs.check_for_fcp(filename) is True: print(' - A 720/576 file with no Pixel Aspect Ratio and scan type metadata has been detected.') ffv1_command += [ '-vf', 'setfield=tff, setdar=4/3' ] print(' - -vf setfield=tff, setdar=4/3 will be added to the FFmpeg command.') ffprobe_dict = ififuncs.get_ffprobe_dict(filename) # let's stipulate the colour metadata if not present for SD PAL material. if not ififuncs.get_colour_metadata(ffprobe_dict): ffv1_command += ['-color_primaries', 'bt470bg', '-color_trc', 'bt709', '-colorspace', 'bt470bg' ] ffv1_command += [ output, '-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio fmd5 ] print(ffv1_command) subprocess.call(ffv1_command, env=ffv1_env_dict) return output, output_uuid, fmd5, ffv1_logfile
def short_test(images): ''' Perform a test on the first 24 frames that will encode via Rawcooked, then decode back to the original form, and the whole file checksums of the original 24 frames and the restored 24 frames are compared. ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid) os.makedirs(temp_dir) # Only analyse the first 24 frames. for image in sorted(os.listdir(images))[:24]: full_path = os.path.join(images, image) shutil.copy(full_path, temp_dir) mkv_uuid = ififuncs.create_uuid() mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv') subprocess.call(['rawcooked', temp_dir, '-c:a', 'copy', '-o', mkv_file]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) subprocess.call(['rawcooked', mkv_file]) rawcooked_dir = mkv_file + '.RAWcooked' restored_dir = os.path.join(rawcooked_dir, temp_uuid) restored_manifest = os.path.join(restored_dir, '456.md5') ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir) judgement = ififuncs.diff_textfiles(converted_manifest, restored_manifest) print((' - Deleting temp directory %s' % temp_dir)) try: shutil.rmtree(temp_dir) except WindowsError: print('Sorry, we do not have permission to delete these files') print((' - Deleting temp reversibility directory %s' % rawcooked_dir)) try: shutil.rmtree(rawcooked_dir) except WindowsError: print('Sorry, we do not have permission to delete these files') print((' - Deleting temp FFV1/MKV %s' % mkv_file)) os.remove(mkv_file) return judgement
def make_folder_path(path, args, object_entry): ''' Generates objects/logs/metadata/UUID folder structure in output. Returns the path. ''' if not args.u: representation_uuid = ififuncs.create_uuid() else: representation_uuid = args.u oe_path = os.path.join(path, object_entry) path = os.path.join(oe_path, representation_uuid) print path ififuncs.make_folder_structure(path) return path
def make_folder_path(path, args, object_entry): ''' Generates objects/logs/metadata/UUID folder structure in output directory. Asks user for UUID if it's not supplied in an arg. Adds a workaround for special collections workflows. Returns the path. UNITTEST - does path exist ''' if not args.u: representation_uuid = ififuncs.create_uuid() else: representation_uuid = args.u if args.sc: oe_path = args.o else: oe_path = os.path.join(path, object_entry) path = os.path.join(oe_path, representation_uuid) ififuncs.make_folder_structure(path) return path
def make_folders(parent, args): ''' Actually make the folders! ''' output = args.o parent_folder = os.path.join(output, parent) os.makedirs(parent_folder) uuid_folder = os.path.join(parent_folder, ififuncs.create_uuid()) logs_dir = os.path.join(uuid_folder, 'logs') objects_dir = os.path.join(uuid_folder, 'objects') metadata_dir = os.path.join(uuid_folder, 'metadata') os.makedirs(logs_dir) os.makedirs(objects_dir) os.makedirs(metadata_dir) for folder in [logs_dir, objects_dir, metadata_dir]: print folder os.chdir(folder) os.makedirs('image') os.makedirs('audio')
def reversibility_verification(objects, source_manifest, reversibility_dir): ''' Restore the MKV back to DPX, create a checksum, and compare to source DPX checksums. Return a value of lossy or lossless. ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(reversibility_dir, temp_uuid) os.makedirs(temp_dir) for ffv1_mkv in objects: subprocess.call(['rawcooked', ffv1_mkv, '-o', temp_dir]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) judgement = ififuncs.diff_textfiles(converted_manifest, source_manifest) print((' - Deleting temp directory %s' % temp_dir)) try: shutil.rmtree(temp_dir) except WindowsError: print('Unable to delete temp directory, sorry!') return judgement
def run_loop(args): ''' This will only process one sequence. Batch processing will come later. ''' current_dir = os.path.dirname(os.path.abspath(sys.argv[0])) if args.user: user = args.user else: user = ififuncs.get_user() object_entry = ififuncs.get_object_entry() log_name_source = os.path.join( args.o, '%s_seq2ffv1_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S") ) ififuncs.generate_log(log_name_source, 'seq2ffv1.py started.') ififuncs.generate_log( log_name_source, 'eventDetail=seq2ffv1.py %s' % ififuncs.get_script_version('seq2ffv1.py')) ififuncs.generate_log( log_name_source, 'Command line arguments: %s' % args ) ififuncs.generate_log( log_name_source, 'EVENT = agentName=%s' % user ) uuid = ififuncs.create_uuid() verdicts = [] multi_reeler = False source_directory = args.i images = ififuncs.get_image_sequence_files(source_directory) if images == 'none': print('no images found in directory - checking for multi-reel sequence') images = ififuncs.check_multi_reel(source_directory) multi_reeler = True if images == 'none': sys.exit() # this is checking for a single reeler. else: images = [source_directory] reel_number = 1 objects = [] short_test_reports = [] rawcooked_logfiles = [] for reel in images: short_test_reports.append(short_test(reel)) for i in short_test_reports: print((' - 24 frame reversibility test for %s is %s' % (os.path.basename(reel), i))) if i == 'lossy': print('It appears that this sequence is not reversible - exiting') sys.exit() time.sleep(2) # check for a/b rolls if reel[-1] in ['a', 'b']: reel_number = reel[-2] ffv1_path, source_abspath, args, log_name_source, normalisation_tool, rawcooked_logfile = make_ffv1( reel, args, log_name_source, reel_number, uuid, multi_reeler ) objects.append(ffv1_path) rawcooked_logfiles.append(rawcooked_logfile) # check for a/b rolls if not reel[-1] in ['a', 'b']: reel_number += 1 judgement = package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir) judgement, sipcreator_log, sipcreator_manifest = judgement verdicts.append([source_directory, judgement]) for verdict in verdicts: print(("%-*s : %s" % (50, args.i, verdict[1]))) ififuncs.generate_log(log_name_source, 'seq2ffv1.py finished.') ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main(): ''' Prints a new UUID to the terminal ''' new_uuid = create_uuid() print new_uuid
def main(args_): ''' Launches the functions that prepare and execute the concatenation. ''' uuid = ififuncs.create_uuid() args = parse_args(args_) print args log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")) ififuncs.generate_log(log_name_source, 'concat.py started.') if args.mov: container = 'mov' else: container = 'mkv' ififuncs.generate_log( log_name_source, 'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py')) ififuncs.generate_log( log_name_source, 'Command line arguments: %s' % args ) if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() ififuncs.generate_log( log_name_source, 'EVENT = agentName=%s' % user ) source_uuid_check = '' if os.path.isfile(args.i[0]): source_uuid = ififuncs.get_source_uuid() elif os.path.isdir(args.i[0]): source_uuid_check = ififuncs.check_for_uuid(args) if source_uuid_check == False: source_uuid = ififuncs.get_source_uuid() else: source_uuid = source_uuid_check ififuncs.generate_log( log_name_source, 'Relationship, derivation, has source=%s' % source_uuid ) video_files = args.i concat_file = ififuncs.get_temp_concat('concat_stuff') ififuncs.generate_log( log_name_source, 'concatenation file=%s' % concat_file) if args.r: video_files = recursive_file_list(video_files) video_files = ififuncs.sanitise_filenames(video_files) for source_files in video_files: ififuncs.generate_log( log_name_source, 'source_files = %s' % source_files) make_chapters(video_files) ififuncs.concat_textfile(video_files, concat_file) ififuncs.generate_log( log_name_source, 'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container))) source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid, container) output_file = os.path.join(args.o, '%s.%s' % (uuid, container)) ififuncs.generate_log( log_name_source, 'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container))) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation') validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:') validation_env_dict = ififuncs.set_environment(validation_logfile) output_bitstream_md5 = subprocess.check_output([ 'ffmpeg', '-report', '-i', output_file, '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-' ], env=validation_env_dict).rstrip() ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation') if source_bitstream_md5 == output_bitstream_md5: print 'process appears to be lossless' print source_bitstream_md5, output_bitstream_md5 ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, eventOutcome=pass') else: print 'something went wrong - not lossless!' print source_bitstream_md5,output_bitstream_md5 ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, eventOutcome=fail') if args.nochapters != True: subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt']) ififuncs.generate_log( log_name_source, 'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.') ififuncs.concat_textfile(video_files, concat_file) with open(log_name_source, 'r') as concat_log: concat_lines = concat_log.readlines() if not args.no_sip: sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o]) shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log)) shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log)) logs_dir = os.path.dirname(sipcreator_log) ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile))) ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':'))))) ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main(args_): ''' Retrospectively updates older FFV1/DV packages in order to meet our current packaging requirements. This should allow accession.py and makepbcore.py to run as expected. This script should work on files created by: makeffv1.py dvsip.py loopline.py ''' args = parse_args(args_) user = ififuncs.get_user() new_object_entry = get_numbers(args) for root, _, filenames in os.walk(args.input): if os.path.basename(root)[:2] == 'oe': if len(os.path.basename(root)[2:]) == 4: log_dir = os.path.join(root, 'logs') for files in os.listdir(log_dir): if '.mov_log.log' in files: log = os.path.join(log_dir, files) old_oe_path = root old_oe = os.path.basename(root) manifest = os.path.join(os.path.dirname(root), old_oe + '_manifest.md5') uuid = ififuncs.create_uuid() uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid ififuncs.generate_log(log, 'EVENT = loopline_repackage.py started') ififuncs.generate_log( log, 'eventDetail=loopline_repackage.py %s' % ififuncs.get_script_version('loopline_repackage.py')) ififuncs.generate_log(log, 'Command line arguments: %s' % args) ififuncs.generate_log(log, 'EVENT = agentName=%s' % user) ififuncs.generate_log(log, uuid_event) ififuncs.generate_log( log, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % new_object_entry) old_uuid_path = os.path.join(os.path.dirname(root), uuid) new_oe_path, new_uuid_path = move_files( root, new_object_entry, old_oe_path, old_uuid_path, uuid) updated_lines = update_manifest(manifest, old_oe, uuid) new_manifest = os.path.join(new_oe_path, uuid) + '_manifest.md5' shutil.move(manifest, new_manifest) with open(new_manifest, 'w') as fo: for lines in updated_lines: fo.write(lines) new_logs_path = os.path.join(new_uuid_path, 'logs') for files in os.listdir(new_logs_path): if '.mov_log.log' in files: log = os.path.join(new_logs_path, files) logname = rename_files(new_uuid_path, old_oe, uuid, new_manifest, log) ififuncs.generate_log( logname, 'EVENT = loopline_repackage.py finished') ififuncs.checksum_replace(new_manifest, logname, 'md5') oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1 new_object_entry = 'oe' + str(oe_digits)
def make_ffv1( start_number, source_abspath, output_dirname, args, log_name_source, user ): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' uuid = ififuncs.create_uuid() if args.sip: object_entry = ififuncs.get_object_entry() files_to_move = [] pix_fmt = ififuncs.img_seq_pixfmt( start_number, source_abspath ) temp_dir = tempfile.gettempdir() ffv1_path = os.path.join(output_dirname, uuid + '.mkv') source_textfile = os.path.join( temp_dir, uuid + '_source.framemd5' ) files_to_move.append(source_textfile) # Just perform framemd5 at this stage if args.rawcooked: logfile = os.path.join( temp_dir, '%s_source_framemd5.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" source_framemd5_env_dict = ififuncs.set_environment(logfile) source_framemd5_cmd = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-pix_fmt', pix_fmt, '-f', 'framemd5', source_textfile ] print(source_abspath) rawcooked_logfile = os.path.join( temp_dir, '%s_rawcooked.log' % uuid ) normalisation_tool = ififuncs.get_rawcooked_version() files_to_move.append(rawcooked_logfile) rawcooked_logfile = "\'" + rawcooked_logfile + "\'" env_dict = ififuncs.set_environment(rawcooked_logfile) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path] if args.audio: rawcooked_cmd.extend([args.audio, '-c:a', 'copy']) ffv12dpx = (rawcooked_cmd) else: logfile = os.path.join( temp_dir, '%s_ffv1_transcode.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" env_dict = ififuncs.set_environment(logfile) ffv12dpx = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-strict', '-2', '-c:v', 'ffv1', '-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16', '-pix_fmt', pix_fmt, ffv1_path, '-f', 'framemd5', source_textfile ] normalisation_tool = 'FFmpeg' print(ffv12dpx) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) subprocess.call(ffv12dpx, env=env_dict) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) ffv1_md5 = os.path.join( temp_dir, uuid + '_ffv1.framemd5' ) files_to_move.append(ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image' ) ffv1_fmd5_cmd = [ 'ffmpeg', '-report', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = os.path.join( temp_dir, '%s_ffv1_framemd5.log' % uuid ) files_to_move.append(ffv1_fmd5_logfile) ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) judgement = verify_losslessness(source_textfile, ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement ) if not args.sip: return judgement else: sip_dir = os.path.join( os.path.dirname(ffv1_path), os.path.join(object_entry, uuid) ) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml] sipcreator_cmd = [ '-i', ffv1_path, '-u', uuid, '-quiet', '-move', '-user', user, '-oe', object_entry, '-o', os.path.dirname(ffv1_path) ] sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd) logs_dir = os.path.join(sip_dir, 'logs') metadata_dir = os.path.join(sip_dir, 'metadata') for files in files_to_move: if files.endswith('.log'): shutil.move(files, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(files)) ) elif files.endswith('.framemd5'): shutil.move(files, metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(files)) ) os.remove(dfxml) os.remove(inputtracexml) os.remove(inputxml) return judgement, sipcreator_log, sipcreator_manifest
def main(args_): ''' Retrospectively updates older FFV1/DV packages in order to meet our current packaging requirements. This should allow accession.py and makepbcore.py to run as expected. This script should work on files created by: makeffv1.py dvsip.py loopline.py ''' args = parse_args(args_) user = ififuncs.get_user() new_object_entry = get_numbers(args) filmographic_csv = args.filmographic technical_csv = args.technical filmographic_oe_list = [] filmo_csv_extraction = ififuncs.extract_metadata(filmographic_csv) tech_csv_extraction = ififuncs.extract_metadata(technical_csv) register = make_register() for line_item in filmo_csv_extraction[0]: dictionary = {} oe_number = line_item['Object Entry'].lower() dictionary['title'] = line_item['Title'] if dictionary['title'] == '': dictionary['title'] = '%s - %s' % (line_item['TitleSeries'], line_item['EpisodeNo']) dictionary['uppercase_dashed_oe'] = oe_number.upper() for tech_record in tech_csv_extraction[0]: if tech_record['Reference Number'] == dictionary[ 'uppercase_dashed_oe']: dictionary['source_accession_number'] = tech_record[ 'Accession Number'] dictionary['filmographic_reference_number'] = tech_record[ 'new_ref'] # this transforms OE-#### to oe#### dictionary['old_oe'] = oe_number[:2] + oe_number[3:] filmographic_oe_list.append(dictionary) for oe_package in filmographic_oe_list: for root, _, filenames in os.walk(args.input): if os.path.basename(root) == oe_package['old_oe']: old_oe_path = root old_oe = os.path.basename(root) log_dir = os.path.join(root, 'logs') for files in os.listdir(log_dir): if '.mov_log.log' in files: log = os.path.join(log_dir, files) manifest = os.path.join(os.path.dirname(root), old_oe + '_manifest.md5') uuid = ififuncs.create_uuid() uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid ififuncs.generate_log(log, 'EVENT = loopline_repackage.py started') ififuncs.generate_log( log, 'eventDetail=loopline_repackage.py %s' % ififuncs.get_script_version('loopline_repackage.py')) ififuncs.generate_log(log, 'Command line arguments: %s' % args) ififuncs.generate_log(log, 'EVENT = agentName=%s' % user) ififuncs.generate_log(log, uuid_event) ififuncs.generate_log( log, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % new_object_entry) ififuncs.generate_log( log, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=Filmographic reference number , value=%s' % oe_package['filmographic_reference_number']) oe_package['new_object_entry'] = new_object_entry print('Transforming %s into %s' % (oe_package['old_oe'], oe_package['new_object_entry'])) ififuncs.generate_log( log, 'Relationship, derivation, has source=%s' % oe_package['source_accession_number']) old_uuid_path = os.path.join(os.path.dirname(root), uuid) new_oe_path, new_uuid_path = move_files( root, new_object_entry, old_oe_path, old_uuid_path, uuid) updated_lines = update_manifest(manifest, old_oe, uuid) new_manifest = os.path.join(new_oe_path, uuid) + '_manifest.md5' shutil.move(manifest, new_manifest) with open(new_manifest, 'w') as fo: for lines in updated_lines: fo.write(lines) new_logs_path = os.path.join(new_uuid_path, 'logs') for files in os.listdir(new_logs_path): if '.mov_log.log' in files: log = os.path.join(new_logs_path, files) logname = rename_files(new_uuid_path, old_oe, uuid, new_manifest, log) date_modified, extension = get_date_modified(new_uuid_path) # This normally would be bad practise, but this project only has two formats. MOV/DV and FFv1/MKV if extension == '.mkv': av_format = 'FFV1/PCM/Matroska' elif extension == '.mov': av_format = 'DV/PCM/QuickTime' provenance_string = 'Reproduction of %s' % oe_package[ 'source_accession_number'] ififuncs.append_csv( register, (oe_package['new_object_entry'].upper()[:2] + '-' + oe_package['new_object_entry'][2:], date_modified, '1', av_format, oe_package['title'], 'contact_name', 'Reproduction', '', provenance_string, '', '')) ififuncs.generate_log( logname, 'EVENT = loopline_repackage.py finished') ififuncs.checksum_replace(new_manifest, logname, 'md5') oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1 new_object_entry = 'oe' + str(oe_digits)
def make_ffv1( start_number, source_abspath, output_dirname, root_filename, args, log_name_source ): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' uuid = ififuncs.create_uuid() if not args.no_sip: object_entry = ififuncs.get_object_entry() files_to_move = [] pix_fmt = ififuncs.img_seq_pixfmt( start_number, source_abspath ) temp_dir = tempfile.gettempdir() logfile = os.path.join( temp_dir, '%s_ffv1_transcode.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" env_dict = ififuncs.set_environment(logfile) ffv1_path = os.path.join(output_dirname, uuid + '.mkv') source_textfile = os.path.join( temp_dir, uuid + '_source.framemd5' ) files_to_move.append(source_textfile) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container' ) ffv12dpx = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-strict', '-2', '-c:v', 'ffv1', '-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16', '-pix_fmt', pix_fmt, ffv1_path, '-f', 'framemd5', source_textfile ] ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container' ) print ffv12dpx subprocess.call(ffv12dpx, env=env_dict) ffv1_md5 = os.path.join( temp_dir, uuid + '_ffv1.framemd5' ) files_to_move.append(ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image' ) ffv1_fmd5_cmd = [ 'ffmpeg', '-report', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = os.path.join( temp_dir, '%s_ffv1_framemd5.log' % uuid ) files_to_move.append(ffv1_fmd5_logfile) ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) judgement = verify_losslessness(source_textfile, ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement ) if args.no_sip: return judgement else: sip_dir = os.path.join( os.path.dirname(ffv1_path), os.path.join(object_entry, uuid) ) sipcreator_log, sipcreator_manifest = sipcreator.main([ '-i', ffv1_path, '-u', uuid, '-quiet', '-move', '-user', 'Kieran', '-oe', object_entry, '-o', os.path.dirname(ffv1_path)]) logs_dir = os.path.join(sip_dir, 'logs') metadata_dir = os.path.join(sip_dir, 'metadata') for files in files_to_move: if files.endswith('.log'): shutil.move(files, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(files)) ) elif files.endswith('.framemd5'): shutil.move(files, metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(files)) ) return judgement, sipcreator_log, sipcreator_manifest