def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i if args.d: try: import clairmeta clairmeta_version = clairmeta.__version__ except ImportError: print( 'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround' ) sys.exit() print(args) user = ififuncs.determine_user(args) object_entry = get_object_entry(args) sip_path = make_folder_path(os.path.join(args.o), args, object_entry) uuid, uuid_event = determine_uuid(args, sip_path) new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') if args.d: content_title = create_content_title_text(sip_path, args) ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started') ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')) ififuncs.generate_log(new_log_textfile, 'Command line arguments: %s' % args) ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user) ififuncs.generate_log(new_log_textfile, uuid_event) if not args.sc: ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry) metadata_dir = os.path.join(sip_path, 'metadata') supplemental_dir = os.path.join(metadata_dir, 'supplemental') logs_dir = os.path.join(sip_path, 'logs') if args.accession: accession_number = ififuncs.get_accession_number() reference_number = ififuncs.get_reference_number() parent = ififuncs.ask_question( 'What is the parent record? eg MV 1234. Enter n/a if this is a born digital acquisition with no parent.' ) donor = ififuncs.ask_question( 'Who is the source of acquisition, as appears on the donor agreement? This will not affect Reproductions.' ) depositor_reference = ififuncs.ask_question( 'What is the donor/depositor number? This will not affect Reproductions.' ) acquisition_type = ififuncs.get_acquisition_type('') donation_date = ififuncs.ask_question( 'When was the donation date in DD/MM/YYYY format? Eg. 31/12/1999 - Unfortunately this is NOT using ISO 8601.' ) if args.zip: inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls( inputs[0], args.o, uuid, new_log_textfile) if args.manifest: shutil.copy( args.manifest, args.manifest.replace('_manifest.md5', '_manifest-md5.txt')) source_manifest = args.manifest.replace('_manifest.md5', '_manifest-md5.txt') else: source_manifest = os.path.join( args.o, os.path.basename(args.i[0]) + '_manifest-md5.txt') ififuncs.generate_log( new_log_textfile, 'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP' ) ififuncs.hashlib_manifest(args.i[0], source_manifest, os.path.dirname(args.i[0])) ififuncs.generate_log( new_log_textfile, 'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP' ) ififuncs.generate_log( new_log_textfile, 'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s' % inputs[0]) makezip_judgement, zip_file = makezip.main([ '-i', inputs[0], '-o', os.path.join(sip_path, 'objects'), '-basename', uuid + '.zip' ]) ififuncs.generate_log( new_log_textfile, 'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, eventDetail=Source object packed into=%s' % zip_file) if makezip_judgement is None: judgement = 'lossless' else: judgement = makezip_judgement ififuncs.generate_log( new_log_textfile, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s' % judgement) ififuncs.generate_log( new_log_textfile, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s' % judgement) else: log_names = move_files(inputs, sip_path, args, user) ififuncs.get_technical_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest(metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir) if args.sc: normalise_objects_manifest(sip_path) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) if args.zip: ififuncs.generate_log( new_log_textfile, 'EVENT = Message Digest Calculation, status=started, eventType=message digest calculation, eventDetail=%s module=hashlib' % zip_file) ififuncs.manifest_update(new_manifest_textfile, zip_file) ififuncs.generate_log( new_log_textfile, 'EVENT = Message Digest Calculation, status=finished, eventType=message digest calculation, eventDetail=%s module=hashlib' % zip_file) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append(logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir))) if args.supplement: os.makedirs(supplemental_dir) supplement_cmd = [ '-i', args.supplement, '-user', user, '-new_folder', supplemental_dir, os.path.dirname(sip_path), '-copy' ] package_update.main(supplement_cmd) if args.zip: os.makedirs(supplemental_dir) supplement_cmd = [ '-i', [inputxml, inputtracexml, dfxml, source_manifest], '-user', user, '-new_folder', supplemental_dir, os.path.dirname(sip_path), '-copy' ] package_update.main(supplement_cmd) if args.sc: print('Generating Digital Forensics XML') dfxml = accession.make_dfxml(args, sip_path, uuid) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)) ififuncs.manifest_update(new_manifest_textfile, dfxml) sha512_log = manifest.main([sip_path, '-sha512', '-s']) sha512_manifest = os.path.join(os.path.dirname(sip_path), uuid + '_manifest-sha512.txt') ififuncs.merge_logs_append(sha512_log, new_log_textfile, new_manifest_textfile) ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512') os.remove(sha512_log) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: if 'log_names' in locals(): log_report(log_names) finish = datetime.datetime.now() print('\n- %s ran this script at %s and it finished at %s' % (user, start, finish)) if args.d: process_dcp(sip_path, content_title, args, new_manifest_textfile, new_log_textfile, metadata_dir, clairmeta_version) if args.accession: register = accession.make_register() filmographic_dict = ififuncs.extract_metadata(args.csv)[0] for filmographic_record in filmographic_dict: if filmographic_record['Reference Number'].lower( ) == reference_number.lower(): if filmographic_record['Title'] == '': title = filmographic_record[ 'TitleSeries'] + '; ' + filmographic_record['EpisodeNo'] else: title = filmographic_record['Title'] oe_register = make_oe_register() ififuncs.append_csv( oe_register, (object_entry.upper()[:2] + '-' + object_entry[2:], donation_date, '1', '', title, donor, acquisition_type[1], accession_number, 'Representation of %s|Reproduction of %s' % (reference_number, parent), '')) accession_cmd = [ os.path.dirname(sip_path), '-user', user, '-f', '-number', accession_number, '-reference', reference_number, '-register', register, '-csv', args.csv, '-pbcore' ] if not parent.lower() == 'n/a': accession_cmd.extend(['-parent', parent]) accession_cmd.extend(['-donor', donor]) accession_cmd.extend(['-depositor_reference', depositor_reference]) accession_cmd.extend(['-acquisition_type', acquisition_type[2]]) accession_cmd.extend(['-donation_date', donation_date]) print(accession_cmd) accession.main(accession_cmd) return new_log_textfile, new_manifest_textfile
def package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir): ''' Package the MKV using sipcreator.py ''' sip_dir = os.path.join( args.o, os.path.join(object_entry, uuid) ) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(source_abspath, args.o, uuid, log_name_source) source_manifest = os.path.join( args.o, os.path.basename(args.i) + '_manifest-md5.txt' ) ififuncs.generate_log( log_name_source, 'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files' ) if multi_reeler: ififuncs.hashlib_manifest(args.i, source_manifest, args.i) else: ififuncs.hashlib_manifest(args.i, source_manifest, os.path.dirname(args.i)) ififuncs.generate_log( log_name_source, 'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files' ) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibility of %s back to its original form, followed by checksum verification using %s ' % (normalisation_tool, objects, source_manifest) ) if args.reversibility_dir: reversibility_dir = args.reversibility_dir else: reversibility_dir = args.o judgement = reversibility_verification(objects, source_manifest, reversibility_dir) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibilty of %s back to its original form, followed by checksum verification using %s , eventOutcome=%s' % (normalisation_tool, objects, source_manifest, judgement) ) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml, source_manifest] if args.supplement: supplement_cmd.extend(args.supplement) sipcreator_cmd = [ '-i', ] for i in objects: sipcreator_cmd.append(i) sipcreator_cmd += [ '-u', uuid, '-quiet', '-move', '-user', user, '-oe', object_entry, '-o', args.o ] sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd) logs_dir = os.path.join(sip_dir, 'logs') for files in os.listdir(logs_dir): if files.endswith('.md5'): deletefiles.main(['-i', os.path.join(logs_dir, files), '-uuid_path', sip_dir, '-user', user]) for rawcooked_logfile in rawcooked_logfiles: rawcooked_logfile = rawcooked_logfile.replace('\'', '') shutil.move(rawcooked_logfile, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(rawcooked_logfile)) ) metadata_dir = os.path.join(sip_dir, 'metadata') os.chdir(current_dir) shutil.copy(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'film_scan_aip_documentation.txt'), metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, 'film_scan_aip_documentation.txt') ) os.remove(dfxml) os.remove(inputtracexml) os.remove(inputxml) return judgement, sipcreator_log, sipcreator_manifest
def main(args_): print('\n - Normalise.py started') args = parse_args(args_) print(args) source = args.i output_folder = args.o file_list = ififuncs.get_video_files(source) if args.sip: if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() oe_digits = int(object_entry.replace('oe', '')) for filename in file_list: log_name_source = os.path.join( args.o, '%s_normalise_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S")) ififuncs.generate_log(log_name_source, 'normalise.py started.') ififuncs.generate_log(log_name_source, 'Command line arguments: %s' % args) ififuncs.generate_log(log_name_source, 'EVENT = agentName=%s' % user) print('\n - Processing: %s' % filename) ififuncs.generate_log( log_name_source, 'EVENT = Normalization, status=started, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object to be normalised=%s' % filename) output, output_uuid, fmd5, ffv1_logfile = normalise_process( filename, output_folder) ififuncs.generate_log( log_name_source, 'EVENT = Normalization, status=finished, eventType=Normalization, agentName=ffmpeg, eventDetail=Source object normalised into=%s' % output) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls( filename, output_folder, output_uuid, log_name_source) fmd5_logfile, fmd5ffv1, verdict = verify_losslessness( output_folder, output, output_uuid, fmd5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation, eventOutcome=%s' % verdict) if args.sip: object_entry_complete = 'oe' + str(oe_digits) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml] sipcreator_cmd = [ '-i', output, '-move', '-u', output_uuid, '-user', user, '-oe', object_entry_complete, '-o', args.o ] if args.supplement: supplement_cmd.extend(args.supplement) sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main( sipcreator_cmd) metadata_dir = os.path.join( os.path.dirname(os.path.dirname(sipcreator_log)), 'metadata') shutil.move(fmd5, metadata_dir) shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log)) shutil.move(fmd5ffv1, metadata_dir) shutil.move( ffv1_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log)) logs_dir = os.path.dirname(sipcreator_log) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(fmd5))) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(fmd5ffv1))) ififuncs.manifest_update( sipcreator_manifest, os.path.join( logs_dir, os.path.basename( ffv1_logfile.replace('\\\\', '\\').replace('\:', ':')))) ififuncs.manifest_update( sipcreator_manifest, os.path.join( logs_dir, os.path.basename( fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')))) ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest) os.remove(dfxml) os.remove(inputxml) os.remove(inputtracexml) oe_digits += 1
def make_ffv1( start_number, source_abspath, output_dirname, args, log_name_source, user ): ''' This launches the image sequence to FFV1/Matroska process as well as framemd5 losslessness verification. ''' uuid = ififuncs.create_uuid() if args.sip: object_entry = ififuncs.get_object_entry() files_to_move = [] pix_fmt = ififuncs.img_seq_pixfmt( start_number, source_abspath ) temp_dir = tempfile.gettempdir() ffv1_path = os.path.join(output_dirname, uuid + '.mkv') source_textfile = os.path.join( temp_dir, uuid + '_source.framemd5' ) files_to_move.append(source_textfile) # Just perform framemd5 at this stage if args.rawcooked: logfile = os.path.join( temp_dir, '%s_source_framemd5.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" source_framemd5_env_dict = ififuncs.set_environment(logfile) source_framemd5_cmd = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-pix_fmt', pix_fmt, '-f', 'framemd5', source_textfile ] print(source_abspath) rawcooked_logfile = os.path.join( temp_dir, '%s_rawcooked.log' % uuid ) normalisation_tool = ififuncs.get_rawcooked_version() files_to_move.append(rawcooked_logfile) rawcooked_logfile = "\'" + rawcooked_logfile + "\'" env_dict = ififuncs.set_environment(rawcooked_logfile) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source' ) rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path] if args.audio: rawcooked_cmd.extend([args.audio, '-c:a', 'copy']) ffv12dpx = (rawcooked_cmd) else: logfile = os.path.join( temp_dir, '%s_ffv1_transcode.log' % uuid ) files_to_move.append(logfile) logfile = "\'" + logfile + "\'" env_dict = ififuncs.set_environment(logfile) ffv12dpx = [ 'ffmpeg', '-report', '-f', 'image2', '-framerate', '24', '-start_number', start_number, '-i', source_abspath, '-strict', '-2', '-c:v', 'ffv1', '-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16', '-pix_fmt', pix_fmt, ffv1_path, '-f', 'framemd5', source_textfile ] normalisation_tool = 'FFmpeg' print(ffv12dpx) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) subprocess.call(ffv12dpx, env=env_dict) ififuncs.generate_log( log_name_source, 'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container' % normalisation_tool ) ffv1_md5 = os.path.join( temp_dir, uuid + '_ffv1.framemd5' ) files_to_move.append(ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image' ) ffv1_fmd5_cmd = [ 'ffmpeg', '-report', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = os.path.join( temp_dir, '%s_ffv1_framemd5.log' % uuid ) files_to_move.append(ffv1_fmd5_logfile) ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) judgement = verify_losslessness(source_textfile, ffv1_md5) ififuncs.generate_log( log_name_source, 'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement ) if not args.sip: return judgement else: sip_dir = os.path.join( os.path.dirname(ffv1_path), os.path.join(object_entry, uuid) ) inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source) supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml] sipcreator_cmd = [ '-i', ffv1_path, '-u', uuid, '-quiet', '-move', '-user', user, '-oe', object_entry, '-o', os.path.dirname(ffv1_path) ] sipcreator_cmd.extend(supplement_cmd) sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd) logs_dir = os.path.join(sip_dir, 'logs') metadata_dir = os.path.join(sip_dir, 'metadata') for files in files_to_move: if files.endswith('.log'): shutil.move(files, logs_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(logs_dir, os.path.basename(files)) ) elif files.endswith('.framemd5'): shutil.move(files, metadata_dir) ififuncs.manifest_update( sipcreator_manifest, os.path.join(metadata_dir, os.path.basename(files)) ) os.remove(dfxml) os.remove(inputtracexml) os.remove(inputxml) return judgement, sipcreator_log, sipcreator_manifest