def get_object_entry(args): ''' Figures out which OE number to use and performs some basic validation. UNITTEST - use the existing ifs to perform some True/False tests. ''' if not args.sc: if args.oe: if args.oe[:2] != 'oe': print( 'First two characters must be \'oe\' and last four characters must be four digits' ) object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in list(range(4, 6)): print( 'First two characters must be \'oe\' and last four characters must be four digits' ) object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print( 'First two characters must be \'oe\' and last four characters must be four digits' ) else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() else: object_entry = 'not_applicable' return object_entry
def get_numbers(args): ''' Figure out the first OE number and how to increment per package. ''' if args.start_number: if args.start_number[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.start_number[2:]) != 4: print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.start_number[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.start_number else: object_entry = ififuncs.get_object_entry() object_entry_digits = int(object_entry[2:]) new_object_entry = 'oe' + str(object_entry_digits) return new_object_entry
def main(args_): print('\n - Normalise.py started') args = parse_args(args_) print(args) source = args.i output_folder = args.o file_list = ififuncs.get_video_files(source) if args.sip: if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() oe_digits = int(object_entry.replace('oe', '')) for filename in file_list: log_name_source = os.path.join( args.o, '%s_normalise_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")) ififuncs.generate_log(log_name_source, 'normalise.py started.') ififuncs.generate_log(log_name_source, 'Command line arguments: %s' % args) ififuncs.generate_log(log_name_source, 'EVENT = agentName=%s' % user) print('\n - Processing: %s' % filename) ififuncs.generate_log( log_name_source, 'EVENT = Normalization, status=started, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object to be normalised=%s' % filename) output, output_uuid, fmd5, ffv1_logfile = normalise_process( filename, output_folder) ififuncs.generate_log( log_name_source, 'EVENT = Normalization, status=finished, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object normalised into=%s' % output) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls( filename, output_folder, output_uuid, log_name_source) fmd5_logfile, fmd5ffv1, verdict = verify_losslessness( output_folder, output, output_uuid, fmd5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation, eventOutcome=%s' % verdict) if args.sip: object_entry_complete = 'oe' + str(oe_digits) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml] sipcreator_cmd = [ '-i', output, '-move', '-u', output_uuid, '-user', user, '-oe', object_entry_complete, '-o', args.o ] if args.supplement: supplement_cmd.extend(args.supplement) sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main( sipcreator_cmd) metadata_dir = os.path.join( os.path.dirname(os.path.dirname(sipcreator_log)), 'metadata') shutil.move(fmd5, metadata_dir) shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log)) shutil.move(fmd5ffv1, metadata_dir) shutil.move( ffv1_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log)) logs_dir = os.path.dirname(sipcreator_log) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(fmd5))) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(fmd5ffv1))) ififuncs.manifest_update( sipcreator_manifest, os.path.join( logs_dir, os.path.basename( ffv1_logfile.replace('\\\\', '\\').replace('\:', ':')))) ififuncs.manifest_update( sipcreator_manifest, os.path.join( logs_dir, os.path.basename( fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')))) ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest) os.remove(dfxml) os.remove(inputxml) os.remove(inputtracexml) oe_digits += 1
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i print args if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) != 4: print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid ) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log( new_log_textfile, 'EVENT = sipcreator.py started' ) ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py') ) ififuncs.generate_log( new_log_textfile, 'Command line arguments: %s' % args ) ififuncs.generate_log( new_log_textfile, 'EVENT = agentName=%s' % user ) ififuncs.generate_log( new_log_textfile, uuid_event ) if args.u is False: sys.exit() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry ) metadata_dir = os.path.join(sip_path, 'metadata') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest( metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir ) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append( logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir)) ) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(args, sip_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), os.path.join('objects', content_title).replace("\\", "/") ) return new_log_textfile, new_manifest_textfile
def run_loop(args): ''' This will only process one sequence. Batch processing will come later. ''' current_dir = os.path.dirname(os.path.abspath(sys.argv[0])) if args.user: user = args.user else: user = ififuncs.get_user() object_entry = ififuncs.get_object_entry() log_name_source = os.path.join( args.o, '%s_seq2ffv1_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S") ) ififuncs.generate_log(log_name_source, 'seq2ffv1.py started.') ififuncs.generate_log( log_name_source, 'eventDetail=seq2ffv1.py %s' % ififuncs.get_script_version('seq2ffv1.py')) ififuncs.generate_log( log_name_source, 'Command line arguments: %s' % args ) ififuncs.generate_log( log_name_source, 'EVENT = agentName=%s' % user ) uuid = ififuncs.create_uuid() verdicts = [] multi_reeler = False source_directory = args.i images = ififuncs.get_image_sequence_files(source_directory) if images == 'none': print('no images found in directory - checking for multi-reel sequence') images = ififuncs.check_multi_reel(source_directory) multi_reeler = True if images == 'none': sys.exit() # this is checking for a single reeler. else: images = [source_directory] reel_number = 1 objects = [] short_test_reports = [] rawcooked_logfiles = [] for reel in images: short_test_reports.append(short_test(reel)) for i in short_test_reports: print((' - 24 frame reversibility test for %s is %s' % (os.path.basename(reel), i))) if i == 'lossy': print('It appears that this sequence is not reversible - exiting') sys.exit() time.sleep(2) # check for a/b rolls if reel[-1] in ['a', 'b']: reel_number = reel[-2] ffv1_path, source_abspath, args, log_name_source, normalisation_tool, rawcooked_logfile = make_ffv1( reel, args, log_name_source, reel_number, uuid, multi_reeler ) objects.append(ffv1_path) rawcooked_logfiles.append(rawcooked_logfile) # check for a/b rolls if not reel[-1] in ['a', 'b']: reel_number += 1 judgement = package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir) judgement, sipcreator_log, sipcreator_manifest = judgement verdicts.append([source_directory, judgement]) for verdict in verdicts: print(("%-*s : %s" % (50, args.i, verdict[1]))) ififuncs.generate_log(log_name_source, 'seq2ffv1.py finished.') ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i print args if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started') ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')) ififuncs.generate_log(new_log_textfile, 'Command line arguments: %s' % args) ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user) ififuncs.generate_log(new_log_textfile, uuid_event) if args.u is False: sys.exit() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry) metadata_dir = os.path.join(sip_path, 'metadata') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest(metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append(logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir))) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(args, sip_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), os.path.join('objects', content_title).replace("\\", "/")) return new_log_textfile, new_manifest_textfile
def main(args_): ''' Launches the functions that prepare and execute the concatenation. ''' uuid = ififuncs.create_uuid() args = parse_args(args_) print args log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")) ififuncs.generate_log(log_name_source, 'concat.py started.') if args.mov: container = 'mov' else: container = 'mkv' ififuncs.generate_log( log_name_source, 'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py')) ififuncs.generate_log( log_name_source, 'Command line arguments: %s' % args ) if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() ififuncs.generate_log( log_name_source, 'EVENT = agentName=%s' % user ) source_uuid_check = '' if os.path.isfile(args.i[0]): source_uuid = ififuncs.get_source_uuid() elif os.path.isdir(args.i[0]): source_uuid_check = ififuncs.check_for_uuid(args) if source_uuid_check == False: source_uuid = ififuncs.get_source_uuid() else: source_uuid = source_uuid_check ififuncs.generate_log( log_name_source, 'Relationship, derivation, has source=%s' % source_uuid ) video_files = args.i concat_file = ififuncs.get_temp_concat('concat_stuff') ififuncs.generate_log( log_name_source, 'concatenation file=%s' % concat_file) if args.r: video_files = recursive_file_list(video_files) video_files = ififuncs.sanitise_filenames(video_files) for source_files in video_files: ififuncs.generate_log( log_name_source, 'source_files = %s' % source_files) make_chapters(video_files) ififuncs.concat_textfile(video_files, concat_file) ififuncs.generate_log( log_name_source, 'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container))) source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid, container) output_file = os.path.join(args.o, '%s.%s' % (uuid, container)) ififuncs.generate_log( log_name_source, 'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container))) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation') validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:') validation_env_dict = ififuncs.set_environment(validation_logfile) output_bitstream_md5 = subprocess.check_output([ 'ffmpeg', '-report', '-i', output_file, '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-' ], env=validation_env_dict).rstrip() ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation') if source_bitstream_md5 == output_bitstream_md5: print 'process appears to be lossless' print source_bitstream_md5, output_bitstream_md5 ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, eventOutcome=pass') else: print 'something went wrong - not lossless!' print source_bitstream_md5,output_bitstream_md5 ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, eventOutcome=fail') if args.nochapters != True: subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt']) ififuncs.generate_log( log_name_source, 'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.') ififuncs.concat_textfile(video_files, concat_file) with open(log_name_source, 'r') as concat_log: concat_lines = concat_log.readlines() if not args.no_sip: sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o]) shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log)) shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log)) logs_dir = os.path.dirname(sipcreator_log) ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile))) ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':'))))) ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def make_ffv1( start_number, source_abspath, output_dirname, args, log_name_source, user ): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' uuid = ififuncs.create_uuid() if args.sip: object_entry = ififuncs.get_object_entry() files_to_move = [] pix_fmt = ififuncs.img_seq_pixfmt( start_number, source_abspath ) temp_dir = tempfile.gettempdir() ffv1_path = os.path.join(output_dirname, uuid + '.mkv') source_textfile = os.path.join( temp_dir, uuid + '_source.framemd5' ) files_to_move.append(source_textfile) # Just perform framemd5 at this stage if args.rawcooked: logfile = os.path.join( temp_dir, '%s_source_framemd5.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" source_framemd5_env_dict = ififuncs.set_environment(logfile) source_framemd5_cmd = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-pix_fmt', pix_fmt, '-f', 'framemd5', source_textfile ] print(source_abspath) rawcooked_logfile = os.path.join( temp_dir, '%s_rawcooked.log' % uuid ) normalisation_tool = ififuncs.get_rawcooked_version() files_to_move.append(rawcooked_logfile) rawcooked_logfile = "\'" + rawcooked_logfile + "\'" env_dict = ififuncs.set_environment(rawcooked_logfile) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path] if args.audio: rawcooked_cmd.extend([args.audio, '-c:a', 'copy']) ffv12dpx = (rawcooked_cmd) else: logfile = os.path.join( temp_dir, '%s_ffv1_transcode.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" env_dict = ififuncs.set_environment(logfile) ffv12dpx = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-strict', '-2', '-c:v', 'ffv1', '-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16', '-pix_fmt', pix_fmt, ffv1_path, '-f', 'framemd5', source_textfile ] normalisation_tool = 'FFmpeg' print(ffv12dpx) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) subprocess.call(ffv12dpx, env=env_dict) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) ffv1_md5 = os.path.join( temp_dir, uuid + '_ffv1.framemd5' ) files_to_move.append(ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image' ) ffv1_fmd5_cmd = [ 'ffmpeg', '-report', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = os.path.join( temp_dir, '%s_ffv1_framemd5.log' % uuid ) files_to_move.append(ffv1_fmd5_logfile) ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) judgement = verify_losslessness(source_textfile, ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement ) if not args.sip: return judgement else: sip_dir = os.path.join( os.path.dirname(ffv1_path), os.path.join(object_entry, uuid) ) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml] sipcreator_cmd = [ '-i', ffv1_path, '-u', uuid, '-quiet', '-move', '-user', user, '-oe', object_entry, '-o', os.path.dirname(ffv1_path) ] sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd) logs_dir = os.path.join(sip_dir, 'logs') metadata_dir = os.path.join(sip_dir, 'metadata') for files in files_to_move: if files.endswith('.log'): shutil.move(files, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(files)) ) elif files.endswith('.framemd5'): shutil.move(files, metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(files)) ) os.remove(dfxml) os.remove(inputtracexml) os.remove(inputxml) return judgement, sipcreator_log, sipcreator_manifest
def make_ffv1( start_number, source_abspath, output_dirname, root_filename, args, log_name_source ): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' uuid = ififuncs.create_uuid() if not args.no_sip: object_entry = ififuncs.get_object_entry() files_to_move = [] pix_fmt = ififuncs.img_seq_pixfmt( start_number, source_abspath ) temp_dir = tempfile.gettempdir() logfile = os.path.join( temp_dir, '%s_ffv1_transcode.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" env_dict = ififuncs.set_environment(logfile) ffv1_path = os.path.join(output_dirname, uuid + '.mkv') source_textfile = os.path.join( temp_dir, uuid + '_source.framemd5' ) files_to_move.append(source_textfile) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container' ) ffv12dpx = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-strict', '-2', '-c:v', 'ffv1', '-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16', '-pix_fmt', pix_fmt, ffv1_path, '-f', 'framemd5', source_textfile ] ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container' ) print ffv12dpx subprocess.call(ffv12dpx, env=env_dict) ffv1_md5 = os.path.join( temp_dir, uuid + '_ffv1.framemd5' ) files_to_move.append(ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image' ) ffv1_fmd5_cmd = [ 'ffmpeg', '-report', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = os.path.join( temp_dir, '%s_ffv1_framemd5.log' % uuid ) files_to_move.append(ffv1_fmd5_logfile) ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) judgement = verify_losslessness(source_textfile, ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement ) if args.no_sip: return judgement else: sip_dir = os.path.join( os.path.dirname(ffv1_path), os.path.join(object_entry, uuid) ) sipcreator_log, sipcreator_manifest = sipcreator.main([ '-i', ffv1_path, '-u', uuid, '-quiet', '-move', '-user', 'Kieran', '-oe', object_entry, '-o', os.path.dirname(ffv1_path)]) logs_dir = os.path.join(sip_dir, 'logs') metadata_dir = os.path.join(sip_dir, 'metadata') for files in files_to_move: if files.endswith('.log'): shutil.move(files, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(files)) ) elif files.endswith('.framemd5'): shutil.move(files, metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(files)) ) return judgement, sipcreator_log, sipcreator_manifest
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i if args.d: try: import clairmeta except ImportError: print( 'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround' ) sys.exit() print args if args.user: user = args.user else: user = ififuncs.get_user() if not args.sc: if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() else: object_entry = 'not_applicable' sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started') ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')) ififuncs.generate_log(new_log_textfile, 'Command line arguments: %s' % args) ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user) ififuncs.generate_log(new_log_textfile, uuid_event) if args.u is False: sys.exit() if not args.sc: ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry) metadata_dir = os.path.join(sip_path, 'metadata') supplemental_dir = os.path.join(metadata_dir, 'supplemental') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest(metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir) if args.sc: normalise_objects_manifest(sip_path) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append(logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir))) if args.supplement: os.makedirs(supplemental_dir) supplement_cmd = [ '-i', args.supplement, '-user', user, '-new_folder', supplemental_dir, os.path.dirname(sip_path), '-copy' ] package_update.main(supplement_cmd) if args.sc: print('Generating Digital Forensics XML') dfxml = accession.make_dfxml(args, sip_path, uuid) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)) ififuncs.manifest_update(new_manifest_textfile, dfxml) sha512_log = manifest.main([sip_path, '-sha512', '-s']) sha512_manifest = os.path.join(os.path.dirname(sip_path), uuid + '_manifest-sha512.txt') ififuncs.merge_logs_append(sha512_log, new_log_textfile, new_manifest_textfile) ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512') os.remove(sha512_log) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(sip_path) new_dcp_path = os.path.join('objects', content_title).replace("\\", "/") absolute_dcp_path = os.path.join(sip_path, new_dcp_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), new_dcp_path) ''' a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT) b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT) ''' dcp = DCP(absolute_dcp_path) clairmeta_version = clairmeta.__version__ dcp_dict = dcp.parse() # json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': ')) xml_str = dicttoxml.dicttoxml(dcp_dict, custom_root='ClairmetaProbe', ids=False, attr_type=False) xml_pretty = prettyprint_xml(xml_str) status, report = dcp.check() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s' % (status, report, clairmeta_version)) clairmeta_xml = os.path.join(metadata_dir, '%s_clairmeta.xml' % content_title) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s' % (clairmeta_xml, clairmeta_version)) with open(clairmeta_xml, 'w') as fo: fo.write(xml_pretty) ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile, 'md5') ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml) print status print report print '\n', user, 'ran this script at %s and it finished at %s' % ( start, finish) return new_log_textfile, new_manifest_textfile
def main(args_): '''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) source_folder = args.i print(args) oe_dict = {} user = ififuncs.determine_user(args) if args.oe: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() oe_digits = int(object_entry.replace('oe', '')) for folder in sorted(os.listdir(source_folder)): full_path = os.path.join(source_folder, folder) if os.path.isdir(full_path): try: folder_contents = os.listdir(full_path) except PermissionError: continue object_entry_complete = 'oe' + str(oe_digits) inputs = [] supplements = [] for files in folder_contents: if os.path.splitext(files)[1][1:].lower() in args.object_extension_pattern: inputs.append(os.path.join(full_path, files)) if os.path.splitext(files)[1][1:].lower() in args.supplement_extension_pattern: supplements.append(os.path.join(full_path, files)) if inputs: print(' - Object Entry: %s\n - Inputs: %s\n - Supplements: %s\n' % (object_entry_complete, inputs, supplements)) oe_dict[object_entry_complete] = [inputs, supplements] oe_digits += 1 else: print('Skipping %s as there are no files in this folder that match the -object_extension_pattern' % full_path) if args.dryrun: print('Exiting as you selected -dryrun') sys.exit() logs = [] if args.y: proceed = 'Y' else: proceed = ififuncs.ask_yes_no( 'Do you want to proceed?' ) if proceed == 'Y': for sips in sorted(oe_dict): print(oe_dict[sips]) sipcreator_cmd = ['-i',] for sipcreator_inputs in oe_dict[sips][0]: sipcreator_cmd.append(sipcreator_inputs) sipcreator_cmd += ['-supplement'] for sipcreator_supplements in oe_dict[sips][1]: sipcreator_cmd.append(sipcreator_supplements) sipcreator_cmd += ['-user', user, '-oe', sips, '-o', args.o] if args.rename_uuid: sipcreator_cmd.append('-rename_uuid') if args.zip: sipcreator_cmd.append('-zip') if args.l: sipcreator_cmd.append('-l') print(sipcreator_cmd) sipcreator_log, _ = sipcreator.main(sipcreator_cmd) logs.append(sipcreator_log) for i in logs: if os.path.isfile(i): print(("%-*s : copyit job was a %s" % (50, os.path.basename(i), analyze_log(i))))