def short_test(images, args): ''' Perform a test on the first 24 frames that will encode via Rawcooked, then decode back to the original form, and the whole file checksums of the original 24 frames and the restored 24 frames are compared. maybe all that needs to happen is that 24 frames are copied to a temp location, then the functions run, and use ififuncs to check the sums ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid) os.makedirs(temp_dir) for image in images[:24]: full_path = os.path.join(args.i, image) shutil.copy(full_path, temp_dir) mkv_uuid = ififuncs.create_uuid() mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv') subprocess.call(['rawcooked', temp_dir, '-o', mkv_file]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) subprocess.call(['rawcooked', mkv_file]) rawcooked_dir = mkv_file + '.RAWcooked' restored_dir = os.path.join(rawcooked_dir, temp_uuid) restored_manifest = os.path.join(restored_dir, '456.md5') ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir) ififuncs.diff_textfiles(converted_manifest, restored_manifest)
def run_loop(args, csv_report_filename): ''' Launches a recursive loop to process all images sequences in your subdirectories. ''' for root, _, filenames in os.walk(args.source_directory): source_directory = root total_size = 0 start = datetime.datetime.now() info = make_framemd5(source_directory, 'dpx_framemd5', args) if info == 'none': continue for files in filenames: total_size += os.path.getsize(os.path.join(root, files)) output_dirname = info[0] source_textfile = info[1] image_seq_without_container = info[2] start_number = info[3] container = info[4] dpx_filename = info[5] sequence_length = info[7] output_filename = image_seq_without_container[:-1] logfile = os.path.join(output_dirname, 'logs/%s_ffv1_transcode.log' % output_filename) logfile = "\'" + logfile + "\'" (ffv1_path, ffv1_md5, transcode_time, pix_fmt, width, height, finish, transcode_start, transcode_finish) = make_ffv1(start_number, dpx_filename, output_dirname, output_filename, image_seq_without_container, ififuncs.set_environment(logfile)) comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path)) judgement = diff_textfiles(source_textfile, ffv1_md5) fps = float(sequence_length) / float(transcode_time) checksum_mismatches = [] with open(source_textfile) as source_md5_object: with open(ffv1_md5) as ffv1_md5_object: for (line1), (line2) in itertools.izip( read_lines(source_md5_object), read_lines(ffv1_md5_object)): if line1 != line2: if 'sar' in line1: checksum_mismatches = ['sar'] else: checksum_mismatches.append(1) log_results(checksum_mismatches, csv_report_filename, os.path.basename(output_dirname), judgement, start, finish, transcode_start, transcode_finish, transcode_time, sequence_length, fps, total_size, os.path.getsize(ffv1_path), pix_fmt, container, width, height, comp_ratio)
def verify_losslessness(source_textfile, ffv1_md5): ''' Compares two framemd5 documents in order to determine losslessness. ''' judgement = ififuncs.diff_textfiles(source_textfile, ffv1_md5) checksum_mismatches = [] with open(source_textfile) as source_md5_object: with open(ffv1_md5) as ffv1_md5_object: for (line1), (line2) in zip(ififuncs.read_lines(source_md5_object), ififuncs.read_lines(ffv1_md5_object)): if line1 != line2: if 'sar' in line1: checksum_mismatches = ['sar'] else: checksum_mismatches.append(1) return judgement
def verify_losslessness(source_textfile, ffv1_md5): ''' Compares two framemd5 documents in order to determine losslessness. ''' judgement = ififuncs.diff_textfiles(source_textfile, ffv1_md5) checksum_mismatches = [] with open(source_textfile) as source_md5_object: with open(ffv1_md5) as ffv1_md5_object: for (line1), (line2) in itertools.izip( ififuncs.read_lines( source_md5_object ), ififuncs.read_lines(ffv1_md5_object) ): if line1 != line2: if 'sar' in line1: checksum_mismatches = ['sar'] else: checksum_mismatches.append(1) return judgement
def reversibility_verification(objects, source_manifest, reversibility_dir): ''' Restore the MKV back to DPX, create a checksum, and compare to source DPX checksums. Return a value of lossy or lossless. ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(reversibility_dir, temp_uuid) os.makedirs(temp_dir) for ffv1_mkv in objects: subprocess.call(['rawcooked', ffv1_mkv, '-o', temp_dir]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) judgement = ififuncs.diff_textfiles(converted_manifest, source_manifest) print((' - Deleting temp directory %s' % temp_dir)) try: shutil.rmtree(temp_dir) except WindowsError: print('Unable to delete temp directory, sorry!') return judgement
def short_test(images): ''' Perform a test on the first 24 frames that will encode via Rawcooked, then decode back to the original form, and the whole file checksums of the original 24 frames and the restored 24 frames are compared. ''' temp_uuid = ififuncs.create_uuid() temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid) os.makedirs(temp_dir) # Only analyse the first 24 frames. for image in sorted(os.listdir(images))[:24]: full_path = os.path.join(images, image) shutil.copy(full_path, temp_dir) mkv_uuid = ififuncs.create_uuid() mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv') subprocess.call(['rawcooked', temp_dir, '-c:a', 'copy', '-o', mkv_file]) converted_manifest = os.path.join(temp_dir, '123.md5') ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir) subprocess.call(['rawcooked', mkv_file]) rawcooked_dir = mkv_file + '.RAWcooked' restored_dir = os.path.join(rawcooked_dir, temp_uuid) restored_manifest = os.path.join(restored_dir, '456.md5') ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir) judgement = ififuncs.diff_textfiles(converted_manifest, restored_manifest) print((' - Deleting temp directory %s' % temp_dir)) try: shutil.rmtree(temp_dir) except WindowsError: print('Sorry, we do not have permission to delete these files') print((' - Deleting temp reversibility directory %s' % rawcooked_dir)) try: shutil.rmtree(rawcooked_dir) except WindowsError: print('Sorry, we do not have permission to delete these files') print((' - Deleting temp FFV1/MKV %s' % mkv_file)) os.remove(mkv_file) return judgement
fmd5copy = root_dir + '/metadata/image' shutil.copy(source_textfile,fmd5copy ) image_seq_without_container = info[2] output_parent_directory = info[3] tiff_filename = image_seq_without_container + "%06d.tiff" dpx_filename = image_seq_without_container + "%06d.dpx" logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container env_dict = set_environment(logfile) generate_log(general_log, 'Starting TIFF to DPX transcode') tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname + '/image/dpx_files' '/' + dpx_filename] print tiff2dpx subprocess.call(tiff2dpx,env=env_dict) generate_log(general_log, 'TIFF to DPX transcode complete') parent_basename = os.path.basename(output_dirname) manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5' generate_log(general_log, 'Generating destination manifest via md5deep and storing as %s' % manifest_textfile) other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5') other_textfile = other[1] judgement = diff_textfiles(source_textfile, other_textfile) generate_log(general_log, 'Outcome of transcode was: %s' % judgement) make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile) source_metadata_dir = root_dir + '/metadata/image' shutil.copy(source_textfile, source_metadata_dir + '/%s' % os.path.basename(source_textfile)) finish = datetime.datetime.now() ''' begin premis ''' premis_log(source_parent_dir, source_directory) append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
fmd5copy = root_dir + '/metadata/image' shutil.copy(source_textfile,fmd5copy ) image_seq_without_container = info[2] output_parent_directory = info[3] tiff_filename = image_seq_without_container + "%06d.tiff" dpx_filename = image_seq_without_container + "%06d.dpx" logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container env_dict = set_environment(logfile) generate_log(general_log, 'Starting TIFF to DPX transcode') tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname + '/image/dpx_files' '/' + dpx_filename] print tiff2dpx subprocess.call(tiff2dpx,env=env_dict) generate_log(general_log, 'TIFF to DPX transcode complete') parent_basename = os.path.basename(output_dirname) manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5' generate_log(general_log, 'Generating destination manifest via md5deep and storing as %s' % manifest_textfile) other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5') other_textfile = other[1] judgement = diff_textfiles(source_textfile, other_textfile) generate_log(general_log, 'Outcome of transcode was: %s' % judgement) make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile) source_metadata_dir = root_dir + '/metadata/image' shutil.copy(source_textfile, source_metadata_dir + '/%s' % os.path.basename(source_textfile)) finish = datetime.datetime.now() ''' begin premis ''' premis_log(source_parent_dir, source_directory) append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
transcode_time = transcode_finish_machine_readable - transcode_start_machine_readable parent_basename = os.path.basename(output_dirname) manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5' ffv1_path = output_dirname + '/video/' + output_filename + '.mkv' width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path) height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path ) ffv1_md5 = output_dirname + '/md5/' + image_seq_without_container + 'ffv1.framemd5' ffv1_fmd5_cmd = ['ffmpeg','-i', ffv1_path, '-pix_fmt', pix_fmt,'-f', 'framemd5', ffv1_md5] ffv1_fmd5_logfile = output_dirname + '/logs/%s_ffv1_framemd5.log' % output_filename ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd,env=ffv1_fmd5_env_dict) finish = datetime.datetime.now() ffv1_size = os.path.getsize(ffv1_path) comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path)) judgement = diff_textfiles(source_textfile, ffv1_md5) fps = float(sequence_length) / float(transcode_time) checksum_mismatches = [] with open(source_textfile) as f1: with open(ffv1_md5) as f2: for (lineno1, line1), (lineno2, line2) in itertools.izip( read_non_comment_lines(f1), read_non_comment_lines(f2)): if line1 != line2: if 'sar' in line1: checksum_mismatches = ['sar'] else: checksum_mismatches.append(1) if len(checksum_mismatches) == 0: print 'LOSSLESS' append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time, sequence_length, fps, total_size, ffv1_size, pix_fmt, container, width, height,comp_ratio))
ffv1_path = output_dirname + '/video/' + output_filename + '.mkv' width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path) height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path) ffv1_md5 = output_dirname + '/md5/' + image_seq_without_container + 'ffv1.framemd5' ffv1_fmd5_cmd = [ 'ffmpeg', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5', ffv1_md5 ] ffv1_fmd5_logfile = output_dirname + '/logs/%s_ffv1_framemd5.log' % output_filename ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'" ffv1_fmd5_env_dict = set_environment(ffv1_fmd5_logfile) subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict) finish = datetime.datetime.now() ffv1_size = os.path.getsize(ffv1_path) comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path)) judgement = diff_textfiles(source_textfile, ffv1_md5) fps = float(sequence_length) / float(transcode_time) checksum_mismatches = [] with open(source_textfile) as f1: with open(ffv1_md5) as f2: for (lineno1, line1), (lineno2, line2) in itertools.izip(read_non_comment_lines(f1), read_non_comment_lines(f2)): if line1 != line2: if 'sar' in line1: checksum_mismatches = ['sar'] else: checksum_mismatches.append(1) if len(checksum_mismatches) == 0: print 'LOSSLESS'