def short_test(images, args):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
maybe all that needs to happen is that 24 frames are copied to a temp
location, then the functions run, and use ififuncs to check the sums
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
for image in images[:24]:
full_path = os.path.join(args.i, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
ififuncs.diff_textfiles(converted_manifest, restored_manifest)
def run_loop(args, csv_report_filename):
'''
Launches a recursive loop to process all images sequences in your
subdirectories.
'''
for root, _, filenames in os.walk(args.source_directory):
source_directory = root
total_size = 0
start = datetime.datetime.now()
info = make_framemd5(source_directory, 'dpx_framemd5', args)
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root, files))
output_dirname = info[0]
source_textfile = info[1]
image_seq_without_container = info[2]
start_number = info[3]
container = info[4]
dpx_filename = info[5]
sequence_length = info[7]
output_filename = image_seq_without_container[:-1]
logfile = os.path.join(output_dirname,
'logs/%s_ffv1_transcode.log' % output_filename)
logfile = "\'" + logfile + "\'"
(ffv1_path, ffv1_md5, transcode_time, pix_fmt, width, height, finish,
transcode_start,
transcode_finish) = make_ffv1(start_number, dpx_filename,
output_dirname, output_filename,
image_seq_without_container,
ififuncs.set_environment(logfile))
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
judgement = diff_textfiles(source_textfile, ffv1_md5)
fps = float(sequence_length) / float(transcode_time)
checksum_mismatches = []
with open(source_textfile) as source_md5_object:
with open(ffv1_md5) as ffv1_md5_object:
for (line1), (line2) in itertools.izip(
read_lines(source_md5_object),
read_lines(ffv1_md5_object)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
log_results(checksum_mismatches, csv_report_filename,
os.path.basename(output_dirname), judgement, start, finish,
transcode_start, transcode_finish,
transcode_time, sequence_length, fps, total_size,
os.path.getsize(ffv1_path), pix_fmt, container, width,
height, comp_ratio)
def verify_losslessness(source_textfile, ffv1_md5):
'''
Compares two framemd5 documents in order to determine losslessness.
'''
judgement = ififuncs.diff_textfiles(source_textfile, ffv1_md5)
checksum_mismatches = []
with open(source_textfile) as source_md5_object:
with open(ffv1_md5) as ffv1_md5_object:
for (line1), (line2) in zip(ififuncs.read_lines(source_md5_object),
ififuncs.read_lines(ffv1_md5_object)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
return judgement
def verify_losslessness(source_textfile, ffv1_md5):
'''
Compares two framemd5 documents in order to determine losslessness.
'''
judgement = ififuncs.diff_textfiles(source_textfile, ffv1_md5)
checksum_mismatches = []
with open(source_textfile) as source_md5_object:
with open(ffv1_md5) as ffv1_md5_object:
for (line1), (line2) in itertools.izip(
ififuncs.read_lines(
source_md5_object
), ififuncs.read_lines(ffv1_md5_object)
):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
return judgement
def reversibility_verification(objects, source_manifest, reversibility_dir):
'''
Restore the MKV back to DPX, create a checksum, and compare to source DPX
checksums.
Return a value of lossy or lossless.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(reversibility_dir, temp_uuid)
os.makedirs(temp_dir)
for ffv1_mkv in objects:
subprocess.call(['rawcooked', ffv1_mkv, '-o', temp_dir])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, source_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Unable to delete temp directory, sorry!')
return judgement
def short_test(images):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
# Only analyse the first 24 frames.
for image in sorted(os.listdir(images))[:24]:
full_path = os.path.join(images, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-c:a', 'copy', '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, restored_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp reversibility directory %s' % rawcooked_dir))
try:
shutil.rmtree(rawcooked_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp FFV1/MKV %s' % mkv_file))
os.remove(mkv_file)
return judgement
fmd5copy = root_dir + '/metadata/image'
shutil.copy(source_textfile,fmd5copy )
image_seq_without_container = info[2]
output_parent_directory = info[3]
tiff_filename = image_seq_without_container + "%06d.tiff"
dpx_filename = image_seq_without_container + "%06d.dpx"
logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
env_dict = set_environment(logfile)
generate_log(general_log, 'Starting TIFF to DPX transcode')
tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname + '/image/dpx_files' '/' + dpx_filename]
print tiff2dpx
subprocess.call(tiff2dpx,env=env_dict)
generate_log(general_log, 'TIFF to DPX transcode complete')
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5'
generate_log(general_log, 'Generating destination manifest via md5deep and storing as %s' % manifest_textfile)
other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5')
other_textfile = other[1]
judgement = diff_textfiles(source_textfile, other_textfile)
generate_log(general_log, 'Outcome of transcode was: %s' % judgement)
make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
source_metadata_dir = root_dir + '/metadata/image'
shutil.copy(source_textfile, source_metadata_dir + '/%s' % os.path.basename(source_textfile))
finish = datetime.datetime.now()
'''
begin premis
'''
premis_log(source_parent_dir, source_directory)
append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
fmd5copy = root_dir + '/metadata/image'
shutil.copy(source_textfile,fmd5copy )
image_seq_without_container = info[2]
output_parent_directory = info[3]
tiff_filename = image_seq_without_container + "%06d.tiff"
dpx_filename = image_seq_without_container + "%06d.dpx"
logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
env_dict = set_environment(logfile)
generate_log(general_log, 'Starting TIFF to DPX transcode')
tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname + '/image/dpx_files' '/' + dpx_filename]
print tiff2dpx
subprocess.call(tiff2dpx,env=env_dict)
generate_log(general_log, 'TIFF to DPX transcode complete')
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5'
generate_log(general_log, 'Generating destination manifest via md5deep and storing as %s' % manifest_textfile)
other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5')
other_textfile = other[1]
judgement = diff_textfiles(source_textfile, other_textfile)
generate_log(general_log, 'Outcome of transcode was: %s' % judgement)
make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
source_metadata_dir = root_dir + '/metadata/image'
shutil.copy(source_textfile, source_metadata_dir + '/%s' % os.path.basename(source_textfile))
finish = datetime.datetime.now()
'''
begin premis
'''
premis_log(source_parent_dir, source_directory)
append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
transcode_time = transcode_finish_machine_readable - transcode_start_machine_readable
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5'
ffv1_path = output_dirname + '/video/' + output_filename + '.mkv'
width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path)
height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path )
ffv1_md5 = output_dirname + '/md5/' + image_seq_without_container + 'ffv1.framemd5'
ffv1_fmd5_cmd = ['ffmpeg','-i', ffv1_path, '-pix_fmt', pix_fmt,'-f', 'framemd5', ffv1_md5]
ffv1_fmd5_logfile = output_dirname + '/logs/%s_ffv1_framemd5.log' % output_filename
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd,env=ffv1_fmd5_env_dict)
finish = datetime.datetime.now()
ffv1_size = os.path.getsize(ffv1_path)
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
judgement = diff_textfiles(source_textfile, ffv1_md5)
fps = float(sequence_length) / float(transcode_time)
checksum_mismatches = []
with open(source_textfile) as f1:
with open(ffv1_md5) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1), read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time, sequence_length, fps, total_size, ffv1_size, pix_fmt, container, width, height,comp_ratio))
ffv1_path = output_dirname + '/video/' + output_filename + '.mkv'
width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path)
height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path)
ffv1_md5 = output_dirname + '/md5/' + image_seq_without_container + 'ffv1.framemd5'
ffv1_fmd5_cmd = [
'ffmpeg', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = output_dirname + '/logs/%s_ffv1_framemd5.log' % output_filename
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
finish = datetime.datetime.now()
ffv1_size = os.path.getsize(ffv1_path)
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
judgement = diff_textfiles(source_textfile, ffv1_md5)
fps = float(sequence_length) / float(transcode_time)
checksum_mismatches = []
with open(source_textfile) as f1:
with open(ffv1_md5) as f2:
for (lineno1,
line1), (lineno2,
line2) in itertools.izip(read_non_comment_lines(f1),
read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'