def verify_losslessness(output_folder, output, output_uuid, fmd5):
'''
Verify the losslessness of the process using framemd5.
An additional metadata check should also occur.
'''
verdict = 'undeclared'
fmd5_logfile = os.path.join(output_folder, '%s_framemd5.log' % output_uuid)
fmd5ffv1 = "%s/%s.framemd5" % (output_folder, output_uuid)
print(' - Framemd5s for each frame of your output file will be stored in: %s' % fmd5ffv1)
fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
print(' - FFmpeg will attempt to verify the losslessness of the normalisation by using Framemd5s.')
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', output,
'-report',
'-f', 'framemd5', '-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
checksum_mismatches = ififuncs.diff_framemd5s(fmd5, fmd5ffv1)
if len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print('Image is lossless, but the Pixel Aspect Ratio is different than the source - this may have been intended.')
verdict = 'Image is lossless, but the Pixel Aspect Ratio is different than the source - this may have been intended.'
else:
print 'not lossless'
verdict = 'not lossless'
elif len(checksum_mismatches) > 1:
print 'not lossless'
verdict = 'not lossless'
elif len(checksum_mismatches) == 0:
print 'YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!'
verdict = 'lossless'
return fmd5_logfile, fmd5ffv1, verdict
def make_framemd5(directory, log_filename_alteration, args):
'''
Apparently this makes framemd5s. But it clearly does a lot more.
'''
(basename, ffmpeg_friendly_name, start_number, images, container,
output_dirname) = make_folder_structure(directory, args)
output = output_dirname + '/metadata/%ssource.framemd5' % (basename)
logfile = output_dirname + '/logs/%s%s.log' % (basename,
log_filename_alteration)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
image_seq_without_container = ffmpeg_friendly_name
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
if len(images[0].split("_")[-1].split(".")) > 2:
image_seq_without_container = ffmpeg_friendly_name[:
-1] + ffmpeg_friendly_name[
-1].replace(
'_', '.')
ffmpeg_friendly_name = image_seq_without_container
ffmpeg_friendly_name += number_regex + '%s' % container
framemd5 = [
'ffmpeg', '-start_number', start_number, '-report', '-f', 'image2',
'-framerate', '24', '-i', ffmpeg_friendly_name, '-f', 'framemd5',
output
]
print framemd5
subprocess.call(framemd5, env=env_dict)
info = [
output_dirname, output, image_seq_without_container, start_number,
container, ffmpeg_friendly_name, number_regex,
len(images)
]
return info
def make_ffv1(reel, args, log_name_source, reel_number, uuid, multi_reeler):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
output_dirname = args.o
if multi_reeler:
mkv_basename = uuid + '_reel%s.mkv' % str(reel_number)
else:
mkv_basename = uuid + '.mkv'
ffv1_path = os.path.join(output_dirname, mkv_basename)
rawcooked_logfile = os.path.join(args.o, '%s_rawcooked.log' % mkv_basename)
normalisation_tool = ififuncs.get_rawcooked_version()
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
rawcooked_cmd = [
'rawcooked', reel, '--check', 'full', '-c:a', 'copy', '-o', ffv1_path
]
ffv12dpx = (rawcooked_cmd)
print(ffv12dpx)
if args.zip:
uuid = ififuncs.create_uuid()
# ugly hack until i recfactor. this is the zip_path, not ffv1_path
ffv1_path = os.path.join(output_dirname, uuid + '.zip')
ififuncs.generate_log(
log_name_source,
'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
% reel)
makezip_judgement = makezip.main([
'-i', reel, '-o', output_dirname, '-basename',
os.path.basename(ffv1_path)
])[0]
ififuncs.generate_log(
log_name_source,
'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, Source object packed into=%s'
% ffv1_path)
if makezip_judgement is None:
judgement = 'lossless'
else:
judgement = makezip_judgement
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
if not args.zip:
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool)
return ffv1_path, reel, args, log_name_source, normalisation_tool, rawcooked_logfile
def normalise_process(filename, output_folder):
'''
Begins the actual normalisation process using FFmpeg
'''
output_uuid = ififuncs.create_uuid()
print(' - The following UUID has been generated: %s' % output_uuid)
output = "%s/%s.mkv" % (
output_folder, output_uuid
)
print(' - The normalised file will have this filename: %s' % output)
fmd5 = "%s/%s_source.framemd5" % (
output_folder, os.path.basename(filename)
)
print(' - Framemd5s for each frame of your input file will be stored in: %s' % fmd5)
ffv1_logfile = os.path.join(output_folder, '%s_normalise.log' % output_uuid)
print(' - The FFmpeg logfile for the transcode will be stored in: %s' % ffv1_logfile)
print(' - FFmpeg will begin normalisation now.')
ffv1_env_dict = ififuncs.set_environment(ffv1_logfile)
ffv1_command = [
'ffmpeg',
'-i', filename,
'-c:v', 'ffv1', # Use FFv1 codec
'-g', '1', # Use intra-frame only aka ALL-I aka GOP=1
'-level', '3', # Use Version 3 of FFv1
'-c:a', 'copy', # Copy and paste audio bitsream with no transcoding
'-map', '0',
'-dn',
'-report',
'-slicecrc', '1',
'-slices', '16',
]
if ififuncs.check_for_fcp(filename) is True:
print(' - A 720/576 file with no Pixel Aspect Ratio and scan type metadata has been detected.')
ffv1_command += [
'-vf',
'setfield=tff, setdar=4/3'
]
print(' - -vf setfield=tff, setdar=4/3 will be added to the FFmpeg command.')
ffprobe_dict = ififuncs.get_ffprobe_dict(filename)
# let's stipulate the colour metadata if not present for SD PAL material.
if not ififuncs.get_colour_metadata(ffprobe_dict):
ffv1_command += ['-color_primaries', 'bt470bg', '-color_trc', 'bt709', '-colorspace', 'bt470bg' ]
ffv1_command += [
output,
'-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print(ffv1_command)
subprocess.call(ffv1_command, env=ffv1_env_dict)
return output, output_uuid, fmd5, ffv1_logfile
def run_loop(args, csv_report_filename):
'''
Launches a recursive loop to process all images sequences in your
subdirectories.
'''
for root, _, filenames in os.walk(args.source_directory):
source_directory = root
total_size = 0
start = datetime.datetime.now()
info = make_framemd5(source_directory, 'dpx_framemd5', args)
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root, files))
output_dirname = info[0]
source_textfile = info[1]
image_seq_without_container = info[2]
start_number = info[3]
container = info[4]
dpx_filename = info[5]
sequence_length = info[7]
output_filename = image_seq_without_container[:-1]
logfile = os.path.join(output_dirname,
'logs/%s_ffv1_transcode.log' % output_filename)
logfile = "\'" + logfile + "\'"
(ffv1_path, ffv1_md5, transcode_time, pix_fmt, width, height, finish,
transcode_start,
transcode_finish) = make_ffv1(start_number, dpx_filename,
output_dirname, output_filename,
image_seq_without_container,
ififuncs.set_environment(logfile))
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
judgement = diff_textfiles(source_textfile, ffv1_md5)
fps = float(sequence_length) / float(transcode_time)
checksum_mismatches = []
with open(source_textfile) as source_md5_object:
with open(ffv1_md5) as ffv1_md5_object:
for (line1), (line2) in itertools.izip(
read_lines(source_md5_object),
read_lines(ffv1_md5_object)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
log_results(checksum_mismatches, csv_report_filename,
os.path.basename(output_dirname), judgement, start, finish,
transcode_start, transcode_finish,
transcode_time, sequence_length, fps, total_size,
os.path.getsize(ffv1_path), pix_fmt, container, width,
height, comp_ratio)
def make_sip(video_files):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_mediainfo.xml" % (
metadata_dir, os.path.basename(filename)
)
inputtracexml = "%s/%s_mediatrace.xml" % (
metadata_dir, os.path.basename(filename)
)
fmd5 = "%s/%s.framemd5" % (
metadata_dir, os.path.basename(filename)
)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
fmd5_logfile = log_dir + '/%s_framemd5.log' % filename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', filename,
'-report',
'-f', 'framemd5', '-an',
fmd5
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log,
'makeffv1.py Framemd5 generation of output file completed'
)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
if os.path.isfile(filename):
shutil.move(filename, data_dir)
generate_log(log, 'dvsip.py DV file moved to %s' % data_dir)
generate_log(log, 'dvsip.py MD5 manifest started')
hashlib_manifest(filenoext, manifest, source_parent_dir)
def make_sip(video_files):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = '%s/metadata' % filenoext
log_dir = '%s/logs' % filenoext
data_dir = '%s/objects' % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = '%s/%s_mediainfo.xml' % (metadata_dir,
os.path.basename(filename))
inputtracexml = '%s/%s_mediatrace.xml' % (metadata_dir,
os.path.basename(filename))
fmd5 = '%s/%s.framemd5' % (metadata_dir, os.path.basename(filename))
log = '%s/%s_log.log' % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
fmd5_logfile = log_dir + '/%s_framemd5.log' % filename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i',
filename,
'-report',
'-f',
'framemd5',
'-an',
fmd5
]
print(fmd5_command)
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file completed')
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print('Generating mediainfo xml of input file and saving it in %s'
) % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print('Generating mediatrace xml of input file and saving it in %s'
) % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
if os.path.isfile(filename):
shutil.move(filename, data_dir)
generate_log(log, 'dvsip.py DV file moved to %s' % data_dir)
generate_log(log, 'dvsip.py MD5 manifest started')
hashlib_manifest(filenoext, manifest, source_parent_dir)
def ffmpeg_concat(concat_file, args, uuid, container):
'''
Launch the actual ffmpeg concatenation command.
'''
fmd5_logfile = os.path.join(args.o, '%s_concat.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
print fmd5_logfile
print fmd5_env_dict
cmd = [
'ffmpeg', '-report', '-f', 'concat', '-safe', '0',
'-i', concat_file,
'-c', 'copy', '-map', '0:v', '-map', '0:a?',
os.path.join(args.o, '%s.%s' % (uuid, container)),
'-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy', '-'
]
print cmd
source_bitstream_md5 = subprocess.check_output(
cmd, env=fmd5_env_dict
)
return source_bitstream_md5.rstrip(), fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')
def make_ffv1(start_number, dpx_filename, output_dirname, output_filename,
image_seq_without_container, env_dict):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
pix_fmt = ififuncs.img_seq_pixfmt(start_number,
os.path.abspath(dpx_filename))
ffv12dpx = [
'ffmpeg', '-report', '-f', 'image2', '-framerate', '24',
'-start_number', start_number, '-i',
os.path.abspath(dpx_filename), '-strict', '-2', '-c:v', 'ffv1',
'-level', '3', '-g', '1', '-slicecrc', '1', '-slices', '16',
'-pix_fmt', pix_fmt,
output_dirname + '/objects/' + output_filename + '.mkv'
]
print ffv12dpx
transcode_start = datetime.datetime.now()
transcode_start_machine = time.time()
subprocess.call(ffv12dpx, env=env_dict)
transcode_finish = datetime.datetime.now()
transcode_finish_machine = time.time()
transcode_time = transcode_finish_machine - transcode_start_machine
ffv1_path = output_dirname + '/objects/' + output_filename + '.mkv'
width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path)
height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path)
ffv1_md5 = os.path.join(output_dirname + '/metadata',
image_seq_without_container + 'ffv1.framemd5')
ffv1_fmd5_cmd = [
'ffmpeg', '-i', ffv1_path, '-pix_fmt', pix_fmt, '-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
output_dirname, 'logs/%s_ffv1_framemd5.log' % output_filename)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
finish = datetime.datetime.now()
return (ffv1_path, ffv1_md5, transcode_time, pix_fmt, width, height,
finish, transcode_start, transcode_finish)
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: # loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
# Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir, os.path.basename(filename))
inputtracexml = "%s/%s_source_mediatrace.xml" % (metadata_dir, os.path.basename(filename))
output = "%s/%s.mkv" % (data_dir, os.path.basename(filename))
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
fmd5 = "%s/%s_source.framemd5" % (metadata_dir, os.path.basename(filename))
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, "Input = %s" % filename)
generate_log(log, "Output = %s" % output)
generate_log(log, "makeffv1.py transcode to FFV1 and framemd5 generation of source started.")
ffv1_logfile = log_dir + "/%s_ffv1_transcode.log" % filename
ffv1_env_dict = set_environment(ffv1_logfile)
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
"ffmpeg",
"-i",
filename,
"-c:v",
"ffv1", # Use FFv1 codec
"-g",
"1", # Use intra-frame only aka ALL-I aka GOP=1
"-level",
"3", # Use Version 3 of FFv1
"-c:a",
"copy", # Copy and paste audio bitsream with no transcoding
"-map",
"0",
"-dn",
"-report",
"-slicecrc",
"1",
"-slices",
"16",
output,
"-f",
"framemd5",
"-an", # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5,
]
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(log, "makeffv1.py transcode to FFV1 and framemd5 generation completed.")
generate_log(log, "makeffv1.py Framemd5 generation of output file started.")
fmd5_logfile = log_dir + "/%s_framemd5.log" % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
"ffmpeg", # Create decoded md5 checksums for every frame of the ffv1 output
"-i",
output,
"-report",
"-f",
"framemd5",
"-an",
fmd5ffv1,
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(log, "makeffv1.py Framemd5 generation of output file completed")
source_video_size = get_mediainfo("source_video_size", "--inform=General;%FileSize%", filename)
ffv1_video_size = get_mediainfo("ffv1_video_size", "--inform=General;%FileSize%", output)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1), read_non_comment_lines(f2)
):
if line1 != line2:
if "sar" in line1:
checksum_mismatches = ["sar"]
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print "LOSSLESS"
append_csv(csv_report_filename, (output, "LOSSLESS", source_video_size, ffv1_video_size, compression_ratio))
generate_log(log, "makeffv1.py Transcode was lossless")
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == "sar":
print "Image content is lossless, Pixel Aspect Ratio has been altered"
append_csv(
csv_report_filename,
(output, "LOSSLESS - different PAR", source_video_size, ffv1_video_size, compression_ratio),
)
generate_log(log, "makeffv1.py Image content is lossless, but Pixel Aspect Ratio has been altered")
elif len(checksum_mismatches) > 1:
print "NOT LOSSLESS"
append_csv(
csv_report_filename, (output, "NOT LOSSLESS", source_video_size, ffv1_video_size, compression_ratio)
)
generate_log(log, "makeffv1.py Not Lossless.")
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical. This may be because of a lossy transcode, or a change in metadata, most likely pixel aspect ratio. Please analyse the framemd5 files for source and output."
make_mediainfo(inputxml, "mediaxmlinput", filename)
make_mediainfo(outputxml, "mediaxmloutput", output)
make_mediatrace(inputtracexml, "mediatracexmlinput", filename)
make_mediatrace(outputtracexml, "mediatracexmloutput", output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest_path = os.path.join(source_parent_dir, filenoext)
manifest = "%s/%s_manifest.md5" % (manifest_path, filenoext)
generate_log(log, "makeffv1.py MD5 manifest started")
hashlib_manifest(filenoext, manifest, manifest_path)
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (
metadata_dir, os.path.basename(filename)
)
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename)
)
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0]
)
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
fmd5 = "%s/%s_source.framemd5" % (
metadata_dir, os.path.basename(filename)
)
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output(
[
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]
).rstrip()
field_order = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%ScanType%", filename
]
).rstrip()
height = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%Height%",
filename
]
).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i', filename,
'-c:v', 'ffv1', # Use FFv1 codec
'-g', '1', # Use intra-frame only aka ALL-I aka GOP=1
'-level', '3', # Use Version 3 of FFv1
'-c:a', 'copy', # Copy and paste audio bitsream with no transcoding
'-map', '0',
'-dn',
'-report',
'-slicecrc', '1',
'-slices', '16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += [
'-vf',
'setfield=tff, setdar=4/3'
]
ffv1_command += [
output,
'-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation completed.'
)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.'
)
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', output,
'-report',
'-f', 'framemd5', '-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log,
'makeffv1.py Framemd5 generation of output file completed'
)
source_video_size = get_mediainfo(
'source_video_size', "--inform=General;%FileSize%", filename
)
ffv1_video_size = get_mediainfo(
'ffv1_video_size', '--inform=General;%FileSize%', output
)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)
):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(
csv_report_filename, (
output,
'LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(
output,
'LOSSLESS - different PAR',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(
csv_report_filename,
(
output, 'NOT LOSSLESS',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
def main(args_):
'''
Launches the functions that prepare and execute the concatenation.
'''
uuid = ififuncs.create_uuid()
args = parse_args(args_)
print args
log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
ififuncs.generate_log(log_name_source, 'concat.py started.')
if args.mov:
container = 'mov'
else:
container = 'mkv'
ififuncs.generate_log(
log_name_source,
'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
ififuncs.generate_log(
log_name_source,
'Command line arguments: %s' % args
)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
ififuncs.generate_log(
log_name_source,
'EVENT = agentName=%s' % user
)
source_uuid_check = ''
if os.path.isfile(args.i[0]):
source_uuid = ififuncs.get_source_uuid()
elif os.path.isdir(args.i[0]):
source_uuid_check = ififuncs.check_for_uuid(args)
if source_uuid_check == False:
source_uuid = ififuncs.get_source_uuid()
else: source_uuid = source_uuid_check
ififuncs.generate_log(
log_name_source,
'Relationship, derivation, has source=%s' % source_uuid
)
video_files = args.i
concat_file = ififuncs.get_temp_concat('concat_stuff')
ififuncs.generate_log(
log_name_source,
'concatenation file=%s' % concat_file)
if args.r:
video_files = recursive_file_list(video_files)
video_files = ififuncs.sanitise_filenames(video_files)
for source_files in video_files:
ififuncs.generate_log(
log_name_source,
'source_files = %s' % source_files)
make_chapters(video_files)
ififuncs.concat_textfile(video_files, concat_file)
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid, container)
output_file = os.path.join(args.o, '%s.%s' % (uuid, container))
ififuncs.generate_log(
log_name_source,
'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.%s' % (uuid, container)))
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
validation_env_dict = ififuncs.set_environment(validation_logfile)
output_bitstream_md5 = subprocess.check_output([
'ffmpeg', '-report',
'-i', output_file,
'-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
], env=validation_env_dict).rstrip()
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
if source_bitstream_md5 == output_bitstream_md5:
print 'process appears to be lossless'
print source_bitstream_md5, output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=pass')
else:
print 'something went wrong - not lossless!'
print source_bitstream_md5,output_bitstream_md5
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, eventOutcome=fail')
if args.nochapters != True:
subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
ififuncs.generate_log(
log_name_source,
'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
ififuncs.concat_textfile(video_files, concat_file)
with open(log_name_source, 'r') as concat_log:
concat_lines = concat_log.readlines()
if not args.no_sip:
sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
logs_dir = os.path.dirname(sipcreator_log)
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
ififuncs.merge_logs(log_name_source, sipcreator_log, sipcreator_manifest)
def make_ffv1(
start_number,
source_abspath,
output_dirname,
args,
log_name_source,
user
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if args.sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
# Just perform framemd5 at this stage
if args.rawcooked:
logfile = os.path.join(
temp_dir,
'%s_source_framemd5.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
source_framemd5_env_dict = ififuncs.set_environment(logfile)
source_framemd5_cmd = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-pix_fmt', pix_fmt,
'-f', 'framemd5', source_textfile
]
print(source_abspath)
rawcooked_logfile = os.path.join(
temp_dir, '%s_rawcooked.log' % uuid
)
normalisation_tool = ififuncs.get_rawcooked_version()
files_to_move.append(rawcooked_logfile)
rawcooked_logfile = "\'" + rawcooked_logfile + "\'"
env_dict = ififuncs.set_environment(rawcooked_logfile)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
subprocess.call(source_framemd5_cmd, env=source_framemd5_env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of source'
)
rawcooked_cmd = ['rawcooked', os.path.dirname(source_abspath), '-o', ffv1_path]
if args.audio:
rawcooked_cmd.extend([args.audio, '-c:a', 'copy'])
ffv12dpx = (rawcooked_cmd)
else:
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
normalisation_tool = 'FFmpeg'
print(ffv12dpx)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
subprocess.call(ffv12dpx, env=env_dict)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finshed, eventType=Creation, agentName=%s, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
% normalisation_tool
)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if not args.sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(os.path.dirname(source_abspath), args.o, uuid, log_name_source)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml]
sipcreator_cmd = [
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir,
os.path.basename(filename))
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename))
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0])
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir,
outputfilename)
fmd5 = "%s/%s_source.framemd5" % (metadata_dir,
os.path.basename(filename))
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]).rstrip()
field_order = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%ScanType%", filename
]).rstrip()
height = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full', "--Inform=Video;%Height%",
filename
]).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i',
filename,
'-c:v',
'ffv1', # Use FFv1 codec
'-g',
'1', # Use intra-frame only aka ALL-I aka GOP=1
'-level',
'3', # Use Version 3 of FFv1
'-c:a',
'copy', # Copy and paste audio bitsream with no transcoding
'-map',
'0',
'-dn',
'-report',
'-slicecrc',
'1',
'-slices',
'16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += ['-vf', 'setfield=tff, setdar=4/3']
ffv1_command += [
output,
'-f',
'framemd5',
'-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation completed.')
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.')
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
pix_fmt = get_ffmpeg_fmt(filename, 'video')
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i',
output,
'-report',
'-pix_fmt',
pix_fmt,
'-f',
'framemd5',
'-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file completed')
source_video_size = get_mediainfo('source_video_size',
"--inform=General;%FileSize%",
filename)
ffv1_video_size = get_mediainfo('ffv1_video_size',
'--inform=General;%FileSize%', output)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
try:
shutil.copy(sys.argv[0], log_dir)
except IOError:
pass
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename,
(output, 'LOSSLESS', source_video_size, ffv1_video_size,
compression_ratio))
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(output, 'LOSSLESS - different PAR', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(csv_report_filename,
(output, 'NOT LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
def make_ffv1(
start_number,
source_abspath,
output_dirname,
root_filename,
args,
log_name_source
):
'''
This launches the image sequence to FFV1/Matroska process
as well as framemd5 losslessness verification.
'''
uuid = ififuncs.create_uuid()
if not args.no_sip:
object_entry = ififuncs.get_object_entry()
files_to_move = []
pix_fmt = ififuncs.img_seq_pixfmt(
start_number,
source_abspath
)
temp_dir = tempfile.gettempdir()
logfile = os.path.join(
temp_dir,
'%s_ffv1_transcode.log' % uuid
)
files_to_move.append(logfile)
logfile = "\'" + logfile + "\'"
env_dict = ififuncs.set_environment(logfile)
ffv1_path = os.path.join(output_dirname, uuid + '.mkv')
source_textfile = os.path.join(
temp_dir, uuid + '_source.framemd5'
)
files_to_move.append(source_textfile)
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
ffv12dpx = [
'ffmpeg', '-report',
'-f', 'image2',
'-framerate', '24',
'-start_number', start_number,
'-i', source_abspath,
'-strict', '-2',
'-c:v', 'ffv1',
'-level', '3',
'-g', '1',
'-slicecrc', '1',
'-slices', '16',
'-pix_fmt', pix_fmt,
ffv1_path,
'-f', 'framemd5', source_textfile
]
ififuncs.generate_log(
log_name_source,
'EVENT = normalisation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Image sequence normalised to FFV1 in a Matroska container'
)
print ffv12dpx
subprocess.call(ffv12dpx, env=env_dict)
ffv1_md5 = os.path.join(
temp_dir,
uuid + '_ffv1.framemd5'
)
files_to_move.append(ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image'
)
ffv1_fmd5_cmd = [
'ffmpeg', '-report',
'-i', ffv1_path,
'-pix_fmt', pix_fmt,
'-f', 'framemd5',
ffv1_md5
]
ffv1_fmd5_logfile = os.path.join(
temp_dir, '%s_ffv1_framemd5.log' % uuid
)
files_to_move.append(ffv1_fmd5_logfile)
ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
judgement = verify_losslessness(source_textfile, ffv1_md5)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=Frame level checksums of image, eventOutcome=%s' % judgement
)
if args.no_sip:
return judgement
else:
sip_dir = os.path.join(
os.path.dirname(ffv1_path), os.path.join(object_entry, uuid)
)
sipcreator_log, sipcreator_manifest = sipcreator.main([
'-i',
ffv1_path,
'-u',
uuid,
'-quiet',
'-move',
'-user',
'Kieran',
'-oe',
object_entry,
'-o', os.path.dirname(ffv1_path)])
logs_dir = os.path.join(sip_dir, 'logs')
metadata_dir = os.path.join(sip_dir, 'metadata')
for files in files_to_move:
if files.endswith('.log'):
shutil.move(files, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(files))
)
elif files.endswith('.framemd5'):
shutil.move(files, metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, os.path.basename(files))
)
return judgement, sipcreator_log, sipcreator_manifest
