def log_results(manifest, log, parent_dir):
'''
Updates the existing log file. This is copy pasted from validate.py.
Eventally, both functions should be merged and moved into ififuncs.
'''
updated_manifest = []
basename = os.path.basename(manifest).replace('_manifest.md5', '')
sip_dir = parent_dir
logs_dir = os.path.join(sip_dir, 'logs')
logname = logname_check(basename, logs_dir)
logfile = os.path.join(logs_dir, logname)
ififuncs.generate_log(
log, 'EVENT = Logs consolidation - Log from %s merged into %s' %
(log, logfile))
if os.path.isfile(logfile):
with open(log, 'r') as fo:
validate_log = fo.readlines()
with open(logfile, 'ab') as ba:
for lines in validate_log:
ba.write(lines)
with open(manifest, 'r') as manifesto:
manifest_lines = manifesto.readlines()
for lines in manifest_lines:
if os.path.basename(logname) in lines:
lines = lines[:31].replace(
lines[:31], ififuncs.hashlib_md5(logfile)) + lines[32:]
updated_manifest.append(lines)
with open(manifest, 'wb') as fo:
for lines in updated_manifest:
fo.write(lines)
def log_results(manifest, log, parent_dir):
'''
Updates the existing log file. This is copy pasted from validate.py.
Eventally, both functions should be merged and moved into ififuncs.
'''
updated_manifest = []
basename = os.path.basename(manifest).replace('_manifest.md5', '')
sip_dir = parent_dir
logs_dir = os.path.join(sip_dir, 'logs')
logname = logname_check(basename, logs_dir)
logfile = os.path.join(logs_dir, logname)
ififuncs.generate_log(
log,
'EVENT = Logs consolidation - Log from %s merged into %s' % (log, logfile)
)
if os.path.isfile(logfile):
with open(log, 'r') as fo:
validate_log = fo.readlines()
with open(logfile, 'ab') as ba:
for lines in validate_log:
ba.write(lines)
with open(manifest, 'r') as manifesto:
manifest_lines = manifesto.readlines()
for lines in manifest_lines:
if os.path.basename(logname) in lines:
lines = lines[:31].replace(lines[:31], ififuncs.hashlib_md5(logfile)) + lines[32:]
updated_manifest.append(lines)
with open(manifest, 'wb') as fo:
for lines in updated_manifest:
fo.write(lines)
def validate(manifest_dict, manifest, log_name_source, missing_files_list):
'''
Validates the files listed in the checksum manifest.
'''
ififuncs.generate_log(
log_name_source,
'Validating %s ' % manifest
)
error_counter = 0
manifest_directory = os.path.dirname(manifest)
os.chdir(manifest_directory)
error_list = []
for i in sorted(manifest_dict.keys()):
print(('Validating %s' % i))
if 'manifest-sha512.txt' in manifest:
current_hash = ififuncs.hashlib_sha512(i)
else:
current_hash = ififuncs.hashlib_md5(i)
if current_hash == manifest_dict[i]:
print(('%s has validated' % i))
else:
print(('%s has mismatched checksum - %s expected - %s hashed' % (i, manifest_dict[i], current_hash)))
ififuncs.generate_log(
log_name_source,
'%s has mismatched checksum - %s expected - %s hashed' % (i, manifest_dict[i], current_hash)
)
error_list.append('%s has mismatched checksum - %s expected - %s hashed' % (i, manifest_dict[i], current_hash))
error_counter += 1
if error_counter > 0:
print(('\n\n*****ERRORS***********!!!!\n***********\nThe number of mismatched checksums is: %s\n***********\n' % error_counter))
ififuncs.generate_log(
log_name_source,
'The number of mismatched checksums is: %s' % error_counter
)
print('***** List of mismatched files*****')
for i in error_list:
print(i)
elif len(missing_files_list) == 0:
print('All checksums have validated')
ififuncs.generate_log(
log_name_source,
'All checksums have validated'
)
if len(missing_files_list) > 0:
print(('ERRORS - The number of missing files: %s' % len(missing_files_list)))
ififuncs.generate_log(
log_name_source,
'ERRORS - The number of mismatched checksums is: %s' % len(missing_files_list)
)
for i in missing_files_list:
print((('%s is missing') % i))
return error_counter + len(missing_files_list)
def log_results(manifest, log, args):
'''
If a sipcreator type log is found,validate will update the log with the
results.
'''
updated_manifest = []
if 'manifest-sha512.txt' in manifest:
basename = os.path.basename(manifest).replace('_manifest-sha512.txt',
'')
else:
basename = os.path.basename(manifest).replace('_manifest.md5', '')
possible_manifests = [
basename + '_manifest-sha512.txt', basename + '_manifest.md5'
]
logname = basename + '_sip_log.log'
sip_dir = os.path.join(os.path.dirname(args.input), basename)
logs_dir = os.path.join(sip_dir, 'logs')
logfile = os.path.join(logs_dir, logname)
if os.path.isfile(logfile):
with open(log, 'r') as fo:
validate_log = fo.readlines()
with open(logfile, 'a') as ba:
for lines in validate_log:
ba.write(lines)
for possible_manifest in possible_manifests:
if os.path.isfile(possible_manifest):
with open(possible_manifest, 'r') as manifesto:
manifest_lines = manifesto.readlines()
for lines in manifest_lines:
if logname in lines:
if 'manifest-sha512.txt' in possible_manifest:
lines = lines[:127].replace(
lines[:127],
ififuncs.hashlib_sha512(
logfile)) + lines[128:]
elif '_manifest.md5' in possible_manifest:
lines = lines[:31].replace(
lines[:31],
ififuncs.hashlib_md5(logfile)) + lines[32:]
updated_manifest.append(lines)
with open(possible_manifest, 'w') as fo:
for lines in updated_manifest:
fo.write(lines)
updated_manifest = []
def make_h264(filename, args, filter_list):
'''
Launches the actuall ffmpeg process using all the info gleaned so far.
'''
if args.crf:
crf_value = args.crf
else:
crf_value = '23'
if args.o:
output = args.o + '/' + os.path.basename(filename) + "_h264.mov"
else:
output = filename + "_h264.mov"
ffmpeg_args = [
'ffmpeg',
'-i', filename,
]
if args.logo:
ffmpeg_args.extend(['-i', args.logo])
ffmpeg_args += [
'-c:a', 'aac',
'-c:v', 'libx264',
'-pix_fmt', 'yuv420p',
'-crf', crf_value
]
if args.wide:
ffmpeg_args.append('-aspect')
ffmpeg_args.append('16:9')
if not args.map:
ffmpeg_args.append('-map')
ffmpeg_args.append('0:a?')
ffmpeg_args.append('-map')
ffmpeg_args.append('0:v')
if len(filter_list) > 0:
for _filter in filter_list:
ffmpeg_args.append(_filter)
ffmpeg_args.append(output)
print(ffmpeg_args)
subprocess.call(ffmpeg_args)
if args.md5:
manifest = '%s_manifest.md5' % filename
print('Generating md5 sidecar...')
h264_md5 = hashlib_md5(filename)
with open(manifest, 'wb') as fo:
fo.write('%s %s' % (h264_md5, filename))
def log_results(manifest, log, args):
updated_manifest = []
basename = os.path.basename(manifest).replace('_manifest.md5', '')
logname = basename + '_sip_log.log'
sip_dir = os.path.join(os.path.dirname(args.input), basename)
logs_dir = os.path.join(sip_dir, 'logs')
logfile = os.path.join(logs_dir, logname)
if os.path.isfile(logfile):
with open(log, 'r') as fo:
validate_log = fo.readlines()
with open(logfile, 'ab') as ba:
for lines in validate_log:
ba.write(lines)
with open(manifest, 'r') as manifesto:
manifest_lines = manifesto.readlines()
for lines in manifest_lines:
if logname in lines:
lines = lines[:31].replace(
lines[:31], ififuncs.hashlib_md5(logfile)) + lines[32:]
updated_manifest.append(lines)
with open(manifest, 'wb') as fo:
for lines in updated_manifest:
fo.write(lines)
def log_results(manifest, log, args):
updated_manifest = []
basename = os.path.basename(manifest).replace('_manifest.md5', '')
logname = basename + '_sip_log.log'
sip_dir = os.path.join(
os.path.dirname(args.input), basename)
logs_dir = os.path.join(sip_dir, 'logs')
logfile = os.path.join(logs_dir, logname)
if os.path.isfile(logfile):
with open(log, 'r') as fo:
validate_log = fo.readlines()
with open(logfile, 'ab') as ba:
for lines in validate_log:
ba.write(lines)
with open(manifest, 'r') as manifesto:
manifest_lines = manifesto.readlines()
for lines in manifest_lines:
if logname in lines:
lines = lines[:31].replace(lines[:31], ififuncs.hashlib_md5(logfile)) + lines[32:]
updated_manifest.append(lines)
with open(manifest, 'wb') as fo:
for lines in updated_manifest:
fo.write(lines)
def make_h264(filename, args, filter_list):
'''
Launches the actuall ffmpeg process using all the info gleaned so far.
'''
if args.crf:
crf_value = args.crf
else:
crf_value = '23'
if args.o:
output = args.o + '/' + os.path.basename(filename) + "_h264.mov"
else:
output = filename + "_h264.mov"
ffmpeg_args = [
'ffmpeg',
'-i', filename,
'-c:a', 'aac',
'-c:v', 'libx264',
'-pix_fmt', 'yuv420p',
'-crf', crf_value
]
if not args.map:
ffmpeg_args.append('-map')
ffmpeg_args.append('0:a?')
ffmpeg_args.append('-map')
ffmpeg_args.append('0:v')
if len(filter_list) > 0:
for _filter in filter_list:
ffmpeg_args.append(_filter)
ffmpeg_args.append(output)
print ffmpeg_args
subprocess.call(ffmpeg_args)
if args.md5:
manifest = '%s_manifest.md5' % filename
print 'Generating md5 sidecar...'
h264_md5 = hashlib_md5(filename)
with open(manifest, 'wb') as fo:
fo.write('%s %s' % (h264_md5, filename))
def get_bitc(video_files, crf_value, number_of_effects, args, bitc, sidecar):
drawtext_options = []
for filename in video_files:
if args.o:
output = args.o + '/' + filename + "_h264.mov"
elif sidecar == 'proxy_folder':
proxy_folder = os.path.dirname(
os.path.abspath(filename)) + '/proxy'
if not os.path.isdir(proxy_folder):
os.makedirs(proxy_folder)
output = proxy_folder + '/' + filename + "_h264.mov"
else:
output = filename + "_h264.mov"
height = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full', "--Inform=Video;%Height%",
filename
]).rstrip()
pixel_aspect_ratio = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]).rstrip()
ffmpeg_args = [
'ffmpeg', '-i', filename, '-c:a', 'aac', '-c:v', 'libx264',
'-pix_fmt', 'yuv420p', '-crf', crf_value
]
if not args.map:
ffmpeg_args.append('-map')
ffmpeg_args.append('0:a?')
ffmpeg_args.append('-map')
ffmpeg_args.append('0:v')
if args.yadif or args.scale or args.player or bitc:
ffmpeg_args.append('-vf')
if bitc:
video_height = float(
getffprobe('video_height', 'stream=height', filename))
video_width = float(
getffprobe('video_width', 'stream=width', filename))
# Calculate appropriate font size
font_size = video_height / 12
watermark_size = video_height / 14
if _platform == "darwin":
font_path = "fontfile=/Library/Fonts/AppleGothic.ttf"
elif _platform == "linux2":
font_path = "fontfile=/usr/share/fonts/truetype/freefont/FreeSerifBold.ttf"
elif _platform == "win32":
font_path = "'fontfile=C\:\\\Windows\\\Fonts\\\\'arial.ttf'"
# Get starting timecode in a raw state that requires processing further on in the script.
timecode_test_raw = getffprobe(
'timecode_test_raw',
'format_tags=timecode:stream_tags=timecode', filename)
get_framerate = getffprobe('get_frame_rate',
'stream=avg_frame_rate', filename)
# This tests if there is actually a timecode present in the file.
if not timecode_test_raw:
# The timecode needs to be phrased in a way unique to each operating system.
# Note the backslashes.
# This section makes up a timecode if none is present in the file.
if _platform == "darwin" or _platform == "linux2":
timecode_test = '01\\\:00\\\:00\\\:00'
elif _platform == "win32":
timecode_test = '01\:00\:00\:00'
else:
# If timecode is present, this will escape the colons
# so that it is compatible with each operating system.
if _platform == "darwin" or _platform == "linux2":
timecode_test = timecode_test_raw.replace(
':', '\\\:').replace('\n', '')
elif _platform == "win32":
timecode_test = timecode_test_raw.replace(
':', '\\:').replace('\n', '').replace('\r', '')
# This removes the new line character from the framemrate.
fixed_framerate = get_framerate.rstrip()
filter_options = ''
#all these prints are just for testing. Will be removed later.
if _platform == "darwin" or _platform == "linux2":
placeholder = ''
drawtext_options = [
"drawtext=%s:fontcolor=white:fontsize=%s:timecode=%s:rate=%s:boxcolor=0x000000AA:box=1:x=(w-text_w)/2:y=h/1.2,drawtext=%s:fontcolor=white:text='IFI IRISH FILM ARCHIVE':x=(w-text_w)/2:y=(h-text_h)/2:fontsize=%s:alpha=0.4"
% (font_path, font_size, timecode_test,
fixed_framerate, font_path, watermark_size)
]
elif _platform == "win32":
drawtext_options = [
"drawtext=%s:fontcolor=white:fontsize=%s:timecode=%s:rate=%s:boxcolor=0x000000AA:box=1:x=(w-text_w)/2:y=h/1.2',drawtext=%s:fontcolor=white:text='IFI IRISH FILM ARCHIVE':x=(w-text_w)/2:y=(h-text_h)/2:fontsize=%s:alpha=0.4'"
% (font_path, font_size, timecode_test,
fixed_framerate, font_path, watermark_size)
]
if args.yadif:
if args.player:
if height == '576':
if pixel_aspect_ratio == '1.000':
ffmpeg_args.append('yadif,scale=768:576')
elif args.clean:
ffmpeg_args.append(' yadif')
else:
drawtext_options[-1] += ',yadif'
if args.scale:
if args.clean:
if number_of_effects > 1:
ffmpeg_args[-1] += (',scale=%s' % width_height)
else:
ffmpeg_args.append('scale=%s' % width_height)
else:
drawtext_options[-1] += (',scale=%s' % width_height)
if len(drawtext_options) > 0:
for i in drawtext_options:
ffmpeg_args.append(i)
ffmpeg_args.append(output)
print ffmpeg_args
subprocess.call(ffmpeg_args)
if args.md5:
manifest = '%s_manifest.md5' % filename
print 'Generating md5 sidecar...'
h264_md5 = hashlib_md5(filename)
with open(manifest, 'wb') as fo:
fo.write('%s %s' % (h264_md5, filename))
def get_bitc(video_files, crf_value, number_of_effects, args, bitc, sidecar):
drawtext_options = []
for filename in video_files:
if args.o:
output = args.o + '/' + filename + "_h264.mov"
elif sidecar == 'proxy_folder':
proxy_folder = os.path.dirname(os.path.abspath(filename)) + '/proxy'
if not os.path.isdir(proxy_folder):
os.makedirs(proxy_folder)
output = proxy_folder + '/' + filename + "_h264.mov"
else:
output = filename + "_h264.mov"
height = subprocess.check_output(['mediainfo','--Language=raw','--Full',"--Inform=Video;%Height%", filename]).rstrip()
pixel_aspect_ratio = subprocess.check_output(['mediainfo','--Language=raw','--Full',"--Inform=Video;%PixelAspectRatio%", filename]).rstrip()
ffmpeg_args = ['ffmpeg',
'-i', filename,
'-c:a', 'aac',
'-c:v', 'libx264',
'-pix_fmt', 'yuv420p',
'-crf', crf_value]
if not args.map:
ffmpeg_args.append('-map')
ffmpeg_args.append('0:a?')
ffmpeg_args.append('-map')
ffmpeg_args.append('0:v')
if args.yadif or args.scale or args.player or bitc :
ffmpeg_args.append('-vf')
if bitc:
video_height = float(getffprobe('video_height','stream=height', filename))
video_width = float(getffprobe('video_width','stream=width', filename))
# Calculate appropriate font size
font_size = video_height / 12
watermark_size = video_height / 14
if _platform == "darwin":
font_path= "fontfile=/Library/Fonts/AppleGothic.ttf"
elif _platform == "linux2":
font_path= "fontfile=/usr/share/fonts/truetype/freefont/FreeSerifBold.ttf"
elif _platform == "win32":
font_path = "'fontfile=C\:\\\Windows\\\Fonts\\\\'arial.ttf'"
# Get starting timecode in a raw state that requires processing further on in the script.
timecode_test_raw = getffprobe('timecode_test_raw','format_tags=timecode:stream_tags=timecode', filename)
get_framerate = getffprobe('get_frame_rate','stream=avg_frame_rate', filename)
# This tests if there is actually a timecode present in the file.
if not timecode_test_raw:
# The timecode needs to be phrased in a way unique to each operating system.
# Note the backslashes.
# This section makes up a timecode if none is present in the file.
if _platform == "darwin" or _platform == "linux2":
timecode_test = '01\\\:00\\\:00\\\:00'
elif _platform == "win32":
timecode_test = '01\:00\:00\:00'
else:
# If timecode is present, this will escape the colons
# so that it is compatible with each operating system.
if _platform == "darwin" or _platform == "linux2":
timecode_test = timecode_test_raw.replace(':', '\\\:').replace('\n', '')
elif _platform == "win32":
timecode_test = timecode_test_raw.replace(':', '\\:').replace('\n', '').replace('\r', '')
# This removes the new line character from the framemrate.
fixed_framerate = get_framerate.rstrip()
filter_options = ''
#all these prints are just for testing. Will be removed later.
if _platform == "darwin" or _platform == "linux2":
placeholder = ''
drawtext_options = ["drawtext=%s:fontcolor=white:fontsize=%s:timecode=%s:rate=%s:boxcolor=0x000000AA:box=1:x=(w-text_w)/2:y=h/1.2,drawtext=%s:fontcolor=white:text='IFI IRISH FILM ARCHIVE':x=(w-text_w)/2:y=(h-text_h)/2:fontsize=%s:alpha=0.4" % (font_path,font_size, timecode_test, fixed_framerate, font_path,watermark_size)]
elif _platform == "win32":
drawtext_options = ["drawtext=%s:fontcolor=white:fontsize=%s:timecode=%s:rate=%s:boxcolor=0x000000AA:box=1:x=(w-text_w)/2:y=h/1.2',drawtext=%s:fontcolor=white:text='IFI IRISH FILM ARCHIVE':x=(w-text_w)/2:y=(h-text_h)/2:fontsize=%s:alpha=0.4'" % (font_path,font_size, timecode_test, fixed_framerate, font_path,watermark_size)]
if args.yadif:
if args.player:
if height =='576':
if pixel_aspect_ratio == '1.000':
ffmpeg_args.append('yadif,scale=768:576')
elif args.clean:
ffmpeg_args.append(' yadif')
else:
drawtext_options[-1] += ',yadif'
if args.scale:
if args.clean:
if number_of_effects > 1:
ffmpeg_args[-1] += (',scale=%s' % width_height)
else:
ffmpeg_args.append('scale=%s' % width_height)
else:
drawtext_options[-1] += (',scale=%s' % width_height)
if len(drawtext_options) > 0:
for i in drawtext_options:
ffmpeg_args.append(i)
ffmpeg_args.append(output)
print ffmpeg_args
subprocess.call(ffmpeg_args)
if args.md5:
manifest = '%s_manifest.md5' % filename
print 'Generating md5 sidecar...'
h264_md5 = hashlib_md5(filename)
with open(manifest,'wb') as fo:
fo.write('%s %s' % (h264_md5, filename))