def log_results(checksum_mismatches, csv_report_filename, parent_basename,
judgement, start, finish, transcode_start, transcode_finish,
transcode_time, sequence_length, fps, total_size, ffv1_size,
pix_fmt, container, width, height, comp_ratio):
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename,
(parent_basename, judgement, start, finish, transcode_start,
transcode_finish, transcode_time, sequence_length, fps,
total_size, ffv1_size, pix_fmt, container, width, height,
comp_ratio))
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
append_csv(
csv_report_filename,
(parent_basename, 'LOSSLESS - different PAR', start, finish,
transcode_start, transcode_finish, transcode_time,
sequence_length, fps, total_size, ffv1_size, pix_fmt,
container, width, height, comp_ratio))
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(csv_report_filename,
(parent_basename, judgement, start, finish, transcode_start,
transcode_finish, transcode_time, sequence_length, fps,
total_size, ffv1_size, pix_fmt, container, width, height,
comp_ratio))
def main():
'''
Launches the commands that generate the CSV error report
'''
args = parse_args()
source = args.input
csv_filename = os.path.join(
ififuncs.make_desktop_logs_dir(),
time.strftime("%Y-%m-%dT%H_%M_%S_videoerrors.csv"))
print('Report stored as %s' % csv_filename)
if not os.path.isfile(csv_filename):
ififuncs.create_csv(
csv_filename,
['filename', 'start_time', 'timestamp', 'error', 'notes'])
for root, _, filenames in os.walk(source):
for filename in filenames:
if filename.endswith('.m2t'):
with open(csv_filename, 'r') as fo:
if not filename in fo.read():
try:
start_time = bitc.getffprobe(
'start_time', 'stream=start_time',
os.path.join(root, filename)).rsplit()[0]
json_output = subprocess.check_output([
'ffprobe', '-sexagesimal',
os.path.join(root, filename), '-show_error',
'-show_log', '16', '-show_frames', '-of',
'json'
])
errors = False
ffprobe_dict = json.loads(json_output)
for values in ffprobe_dict:
for more in ffprobe_dict[values]:
if 'logs' in more:
errors = True
print(more['pkt_pts_time'],
more['logs'])
ififuncs.append_csv(
csv_filename, [
filename, start_time,
more['pkt_pts_time'],
more['logs'], ''
])
if errors == False:
ififuncs.append_csv(csv_filename, [
filename, start_time, 'no errors',
'no errors', ''
])
except subprocess.CalledProcessError:
ififuncs.append_csv(csv_filename, [
filename, start_time,
'script error - process file manually', '', ''
])
print('Report stored as %s' % csv_filename)
ffv1_size = os.path.getsize(ffv1_path)
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
judgement = diff_textfiles(source_textfile, ffv1_md5)
fps = float(sequence_length) / float(transcode_time)
checksum_mismatches = []
with open(source_textfile) as f1:
with open(ffv1_md5) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1), read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time, sequence_length, fps, total_size, ffv1_size, pix_fmt, container, width, height,comp_ratio))
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
append_csv(csv_report_filename, (parent_basename,'LOSSLESS - different PAR',start, finish,transcode_start, transcode_finish,transcode_time,sequence_length, fps,total_size, ffv1_size, pix_fmt, container,width, height,comp_ratio))
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
print csv_report_filename
append_csv(csv_report_filename, (parent_basename,judgement, start, finish,transcode_start, transcode_finish,transcode_time,sequence_length, fps,total_size, ffv1_size, pix_fmt,container, width, height,comp_ratio))
make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
def main(args_):
# if multiple file are present, this script will treat them as a single
# instantiation/representation and get aggregate metadata about the whole
# package. For now, this will be a clumsy implementation - the first file
# will provide most metadata. Things like duration/bitrate/filesize
# will be calculated as a whole.
# Although another way would be that every call is looped, and if
# this could catch files that should not be in the package, eg. a 4:2:2
# file in a 4:2:0 package..
# yup - do it that way!
args = parse_args(args_)
all_files = ififuncs.recursive_file_list(args.input)
silence = True
if args.user:
user = args.user
else:
user = ififuncs.get_user()
acquisition_type = ''
if args.acquisition_type:
acquisition_type = ififuncs.get_acquisition_type(
args.acquisition_type)[0]
for dirs in os.listdir(args.input):
if ififuncs.validate_uuid4(dirs) is None:
instantiationIdentif = dirs
Accession_Number = get_accession_number(args.input)
if args.reference:
Reference_Number = args.reference.upper()
else:
Reference_Number = get_reference_number(args.input)
if args.p:
for root, _, filenames in os.walk(args.input):
if os.path.basename(root) == 'metadata':
metadata_dir = root
elif os.path.basename(root) == 'logs':
logs_dir = root
csv_filename = os.path.join(metadata_dir,
Accession_Number + '_pbcore.csv')
sipcreator_log = os.path.join(logs_dir,
instantiationIdentif + '_sip_log.log')
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
ififuncs.generate_log(
sipcreator_log, 'eventDetail=makepbcore.py %s' %
ififuncs.get_script_version('makepbcore.py'))
ififuncs.generate_log(sipcreator_log,
'Command line arguments: %s' % args)
ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
else:
csv_filename = 'blaa.csv'
print ' - Metadata will be stored in %s' % csv_filename
for filenames in os.listdir(args.input):
if '_manifest.md5' in filenames:
md5_manifest = os.path.join(args.input, filenames)
elif 'manifest-sha512.txt' in filenames:
sha512_manifest = os.path.join(args.input, filenames)
make_csv(csv_filename)
ms = 0
FrameCount = 0
instantFileSize_byte = 0
instantFileSize_gigs = 0
scan_types = []
matrix_list = []
transfer_list = []
colour_primaries_list = []
color_spaces = []
chroma = []
frame_sizes = []
par_list = []
container_list = []
fps_list = []
sample_rate_list = []
track_count_list = []
interlace_list = []
compression_list = []
pix_fmt_list = []
audio_fmt_list = []
audio_codecid_list = []
audio_codec_list = []
au_bitdepth_list = []
video_codecid_list = []
video_codec_version_list = []
video_codec_profile_list = []
timecode_list = []
channels_list = []
for source in all_files:
metadata = subprocess.check_output(
['mediainfo', '--Output=PBCore2', source])
root = etree.fromstring(metadata)
print(' - Analysing %s') % source
pbcore_namespace = root.xpath('namespace-uri(.)')
track_type = root.xpath('//ns:essenceTrackType',
namespaces={'ns': pbcore_namespace})
if len(track_type) > 0:
for track in track_type:
if track.text == 'Video':
essenceTrackEncodvid = get_metadata(
"ns:essenceTrackEncoding", track.getparent(),
pbcore_namespace)
vcodec_attributes = get_attributes(track.getparent(),
pbcore_namespace)
elif track.text == 'Audio':
silence = False
essenceTrackEncod_au = get_metadata(
"//ns:essenceTrackEncoding", track.getparent(),
pbcore_namespace)
audio_codec_list.append(essenceTrackEncod_au)
acodec_attributes = get_attributes(track.getparent(),
pbcore_namespace)
try:
audio_codecid = acodec_attributes['ref']
except KeyError:
audio_codecid = 'n/a'
essenceTrackSampling = ififuncs.get_mediainfo(
'samplerate', '--inform=Audio;%SamplingRate_String%',
source)
sample_rate_list.append(essenceTrackSampling)
essenceBitDepth_au = get_metadata(
"//ns:essenceTrackBitDepth", root, pbcore_namespace)
audio_codecid_list.append(audio_codecid)
au_bitdepth_list.append(essenceBitDepth_au)
channels = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='Channel(s)']",
track.getparent(), pbcore_namespace)
channels_list.append(channels)
ScanType = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ScanType']", root,
pbcore_namespace)
scan_types.append(ScanType)
matrix_coefficients = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='matrix_coefficients']",
root, pbcore_namespace)
timecode_source, starting_timecode = get_timecode(
pbcore_namespace, root, source)
timecode_list.append(starting_timecode)
matrix_list.append(matrix_coefficients)
transfer_characteris = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='transfer_characteristics']",
root, pbcore_namespace)
transfer_list.append(transfer_characteris)
colour_primaries = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='color_primaries']",
root, pbcore_namespace)
colour_primaries_list.append(colour_primaries)
FrameCount += int(
get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='FrameCount']",
root, pbcore_namespace))
instantFileSize_byte += int(
get_metadata("//ns:instantiationFileSize", root, pbcore_namespace))
instantDataRate = round(
float(
ififuncs.get_mediainfo('OverallBitRate',
'--inform=General;%OverallBitRate%',
source)) / 1000 / 1000, 2)
instantTracks = ififuncs.get_number_of_tracks(source)
track_count_list.append(instantTracks)
ms += ififuncs.get_milliseconds(source)
ColorSpace = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ColorSpace']", root,
pbcore_namespace)
color_spaces.append(ColorSpace)
ChromaSubsampling = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ChromaSubsampling']",
root, pbcore_namespace)
chroma.append(ChromaSubsampling)
instantMediaty = get_metadata("//ns:instantiationMediaType", root,
pbcore_namespace)
essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
pbcore_namespace)
frame_sizes.append(essenceFrameSize)
PixelAspectRatio = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='PixelAspectRatio']",
root, pbcore_namespace)
par_list.append(PixelAspectRatio)
instantiationStandar = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Format']", root,
pbcore_namespace)
container_list.append(instantiationStandar)
essenceFrameRate = get_metadata("//ns:essenceTrackFrameRate", root,
pbcore_namespace)
fps_list.append(essenceFrameRate)
essenceAspectRatio = ififuncs.get_mediainfo(
'DAR', '--inform=Video;%DisplayAspectRatio_String%', source)
Interlacement = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Interlacement']",
root, pbcore_namespace)
interlace_list.append(Interlacement)
Compression_Mode = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Compression_Mode']",
root, pbcore_namespace)
colour_range = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='colour_range']",
root, pbcore_namespace)
format_version = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Format_Version']",
root, pbcore_namespace)
app_company_name = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Encoded_Application_CompanyName']",
root, pbcore_namespace)
app_name = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Encoded_Application_Name']",
root, pbcore_namespace)
app_version = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Encoded_Application_Version']",
root, pbcore_namespace)
library_name = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Encoded_Library_Name']",
root, pbcore_namespace)
library_version = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Encoded_Library_Version']",
root, pbcore_namespace)
compression_list.append(Compression_Mode)
instantiationDate_mo = get_metadata(
"//ns:instantiationDate[@dateType='file modification']", root,
pbcore_namespace)
instantDate_other = 'n/a'
instantDate_type = 'n/a'
pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
pix_fmt_list.append(pix_fmt)
audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
audio_fmt_list.append(audio_fmt)
if silence:
audio_codecid = 'n/a'
essenceBitDepth_au = 'n/a'
essenceTrackEncod_au = 'n/a'
essenceTrackSampling = 'n/a'
video_codecid = vcodec_attributes['ref']
video_codecid_list.append(video_codecid)
try:
video_codec_version = vcodec_attributes['version']
except KeyError:
video_codec_version = 'n/a'
try:
video_codec_profile = vcodec_attributes['annotation'][8:]
except KeyError:
video_codec_profile = 'n/a'
video_codec_version_list.append(video_codec_version)
video_codec_profile_list.append(video_codec_profile)
metadata_error = ''
metadata_list = [
scan_types, matrix_list, transfer_list, colour_primaries_list,
color_spaces, chroma, frame_sizes, par_list, container_list, fps_list,
sample_rate_list, track_count_list, interlace_list, compression_list,
pix_fmt_list, audio_fmt_list, audio_codecid_list, audio_codec_list,
au_bitdepth_list, video_codecid_list, video_codec_version_list,
video_codec_profile_list, channels_list, timecode_list
]
for i in metadata_list:
if len(set(i)) > 1:
metadata_error += 'WARNING - Your metadata values are not the same for all files: %s\n' % set(
i)
print metadata_error
if args.p:
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata mismatch - Your metadata values are not the same for all files: %s'
% set(i))
tc = ififuncs.convert_millis(ms)
instantiationDuratio = ififuncs.convert_timecode(25, tc)
if args.donor:
Donor = args.donor
else:
Donor = ''
Edited_By = user
Date_Created = ''
Date_Last_Modified = ''
Film_Or_Tape = 'Digital AV Object'
Date_Of_Donation = ''
reproduction_creator = ''
if args.acquisition_type:
if acquisition_type == 'Reproduction':
Date_Of_Donation = instantiationDate_mo.split('T')[0]
# if a reproduction, then there's no Donor/transfer of title.
Donor = 'n/a'
if ififuncs.find_concat_user(
sipcreator_log) == 'Aoife Fitzmaurice':
reproduction_creator = 'Aoife Fitzmaurice (2016)'
elif ififuncs.find_concat_user(
sipcreator_log) == 'Kieran O\'Leary':
reproduction_creator = 'Kieran O\'Leary (2013)'
Habitat = ''
backup_habitat = ''
Type_Of_Deposit = acquisition_type
if args.depositor_reference:
Depositor_Reference = args.depositor_reference
else:
Depositor_Reference = ''
Master_Viewing = 'Preservation Object'
Language_Version = ''
Condition_Rating = ''
Companion_Elements = ''
TTape_Origin = args.parent
EditedNew = user
FIO = 'In'
CollectionTitle = ''
Created_By = user
instantTimeStart = 'n/a'
instantFileSize_gigs = round(
float(instantFileSize_byte) / 1024 / 1024 / 1024, 3)
instantColors = 'n/a'
instantLanguage = 'n/a'
instantAltMo = 'n/a'
essenceBitDepth_vid = ififuncs.get_mediainfo('duration',
'--inform=Video;%BitDepth%',
source)
instantiationChanCon = 'n/a'
colour_range = colour_range
format_version = format_version
TimeCode_FirstFrame = process_mixed_values(timecode_list)
TimeCode_Source = timecode_source
reproduction_reason = ''
dig_object_descrip = ififuncs.get_digital_object_descriptor(args.input)
ififuncs.append_csv(csv_filename, [
Reference_Number,
Donor,
Edited_By,
Date_Created,
Date_Last_Modified,
Film_Or_Tape,
Date_Of_Donation,
Accession_Number,
Habitat,
backup_habitat,
TTape_Origin,
Type_Of_Deposit,
Depositor_Reference,
Master_Viewing,
Language_Version,
Condition_Rating,
Companion_Elements,
EditedNew,
FIO,
CollectionTitle,
Created_By,
instantiationIdentif,
instantDate_other,
instantDate_type,
instantiationDate_mo,
instantiationStandar,
instantMediaty,
instantFileSize_byte,
instantFileSize_gigs,
instantTimeStart,
instantDataRate,
instantTracks,
instantColors,
instantLanguage,
instantAltMo,
essenceTrackEncodvid,
essenceFrameRate,
essenceTrackSampling,
essenceBitDepth_vid,
essenceFrameSize,
essenceAspectRatio,
essenceTrackEncod_au,
essenceBitDepth_au,
instantiationDuratio,
instantiationChanCon,
PixelAspectRatio,
FrameCount,
ColorSpace,
ChromaSubsampling,
ScanType,
Interlacement,
Compression_Mode,
colour_primaries,
transfer_characteris,
matrix_coefficients,
pix_fmt,
audio_fmt,
audio_codecid,
video_codecid,
video_codec_version,
video_codec_profile,
channels,
colour_range,
format_version,
TimeCode_FirstFrame,
TimeCode_Source,
app_company_name,
app_name,
app_version,
library_name,
library_version,
reproduction_creator,
reproduction_reason,
dig_object_descrip,
])
if args.p:
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
% (csv_filename))
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(md5_manifest, csv_filename)
print ' - Updating %s with %s' % (md5_manifest, csv_filename)
ififuncs.sha512_update(sha512_manifest, csv_filename)
print ' - Updating %s with %s' % (sha512_manifest, csv_filename)
print metadata_error
def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n' )
while permission not in ('Y','y','N','n'):
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n')
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission =='y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename, ('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root,dirnames,filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root# + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root,files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename( master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename( mezzanine_parent_dir) + '_manifest.md5'
master_audio = master_parent_dir + '/objects/audio/' + os.listdir(master_parent_dir + '/objects/audio')[0]
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
source_representation_uuid = premis_description(master_object_dir, master_parent_dir + '/objects/audio', user)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir,audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores= ['ffmpeg','-y','-f','image2','-framerate','24', '-start_number', start_number, '-i', root + '/' + dpx_filename ,'-i', audio_file,'-c:v','prores','-profile:v', '3','-c:a','pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov','-f', 'framemd5', '-an', master_metadata_dir + '/image/' + os.path.basename(master_parent_dir) + '.framemd5', '-c:a', 'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir + '/audio/' + os.path.basename(master_parent_dir) + '.framemd5']
print seq2prores
subprocess.call(seq2prores,env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(mezzanine_file)
items = {"workflow":"seq2prores","oe":'n/a', "filmographic":split_list[0], "sourceAccession":split_list[1], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis, premis_namespace,premisxml, representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(xml_info[2]) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_representation(premisxml, premis_namespace, doc, premis, items,linking_representation_uuids, representation_uuid,sequence )
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid ]
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid ]
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid,prores_event_uuid]
operatorAgent = make_agent(premis,operatorEvents ,user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(premis, 'creation', 'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, ffmpegAgent, operatorAgent ],prores_event_uuid,[representation_uuid], 'outcome')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
tracexml = "%s/%s_mediatrace.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
audio_mediatracexml = "%s/%s_mediatrace.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml,'audiomediaxmlinput',master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml,'audiomediatraceinput',master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(mezzanine_mediainfoxml,'mediaxmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(tracexml,'mediatracexmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest, master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest, mezzanine_parent_dir)
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, operatorAgent],final_sip_manifest_uuid,[representation_uuid], 'source')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename, (os.path.basename( master_parent_dir), start, finish))
'''
def main(args_):
# if multiple file are present, this script will treat them as a single
# instantiation/representation and get aggregate metadata about the whole
# package. For now, this will be a clumsy implementation - the first file
# will provide most metadata. Things like duration/bitrate/filesize
# will be calculated as a whole.
# Although another way would be that every call is looped, and if
# this could catch files that should not be in the package, eg. a 4:2:2
# file in a 4:2:0 package..
# yup - do it that way!
args = parse_args(args_)
all_files = ififuncs.recursive_file_list(args.input)
silence = True
audio_only = True
if args.user:
user = args.user
else:
user = ififuncs.get_user()
acquisition_type = ''
if args.acquisition_type:
acquisition_type = ififuncs.get_acquisition_type(
args.acquisition_type)[0]
instantiationIdentif = ''
for dirs in os.listdir(args.input):
if ififuncs.validate_uuid4(dirs) is None:
instantiationIdentif = dirs
Accession_Number = get_accession_number(args.input)
if args.reference:
Reference_Number = args.reference.upper()
else:
Reference_Number = get_reference_number(args.input)
if args.p:
for root, _, filenames in os.walk(args.input):
if os.path.basename(root) == 'metadata':
metadata_dir = root
elif os.path.basename(root) == 'logs':
logs_dir = root
csv_filename = os.path.join(
metadata_dir,
Accession_Number + '_%s_pbcore.csv' % Reference_Number)
sipcreator_log = os.path.join(logs_dir,
instantiationIdentif + '_sip_log.log')
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
ififuncs.generate_log(
sipcreator_log, 'eventDetail=makepbcore.py %s' %
ififuncs.get_script_version('makepbcore.py'))
ififuncs.generate_log(sipcreator_log,
'Command line arguments: %s' % args)
ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
else:
csv_filename = 'blaa.csv'
print((' - Metadata will be stored in %s' % csv_filename))
for filenames in os.listdir(args.input):
if '_manifest.md5' in filenames:
md5_manifest = os.path.join(args.input, filenames)
elif 'manifest-sha512.txt' in filenames:
sha512_manifest = os.path.join(args.input, filenames)
make_csv(csv_filename)
ms = 0
FrameCount = 0
instantFileSize_byte = 0
instantFileSize_gigs = 0
scan_types = []
matrix_list = []
transfer_list = []
colour_primaries_list = []
color_spaces = []
chroma = []
frame_sizes = []
par_list = []
container_list = []
fps_list = []
sample_rate_list = []
track_count_list = []
interlace_list = []
compression_list = []
pix_fmt_list = []
audio_fmt_list = []
audio_codecid_list = []
audio_codec_list = []
au_bitdepth_list = []
video_codecid_list = []
video_codec_version_list = []
video_codec_profile_list = []
timecode_list = []
channels_list = []
stl = False
subtitle_check = ififuncs.get_digital_object_descriptor(args.input)
if 'STL' in subtitle_check:
stl = True
for source in all_files:
metadata = subprocess.check_output(
['mediainfo', '--Output=PBCore2', source])
new_metadata = subprocess.check_output(
['mediainfo', '--Output=XML', source])
try:
root = etree.fromstring(metadata)
new_root = etree.fromstring(new_metadata)
except lxml.etree.XMLSyntaxError:
print('Windows encoding detected - transforming into utf-8')
root = etree.fromstring(metadata.decode('cp1252').encode('utf-8'))
new_root = etree.fromstring(
new_metadata.decode('cp1252').encode('utf-8'))
print(((' - Analysing %s') % source))
pbcore_namespace = root.xpath('namespace-uri(.)')
mediainfo_namespace = new_root.xpath('namespace-uri(.)')
track_type = root.xpath('//ns:essenceTrackType',
namespaces={'ns': pbcore_namespace})
new_track_type = new_root.xpath('//ns:track',
namespaces={'ns': mediainfo_namespace})
if len(new_track_type) > 0:
for track in new_track_type:
if track.attrib['type'] == 'Video':
audio_only = False
essenceTrackEncodvid = ififuncs.get_metadata(
"ns:Format", track, mediainfo_namespace)
#vcodec_attributes = get_attributes(track.getparent(), pbcore_namespace)
#vcodec_attributes = 'TODO'
video_codecid = ififuncs.get_metadata(
"ns:CodecID", track, mediainfo_namespace)
video_codec_version = ififuncs.get_metadata(
"ns:Format_Version", track, mediainfo_namespace)
video_codec_profile = ififuncs.get_metadata(
"ns:Format_Profile", track, mediainfo_namespace)
video_codec_version_list.append(video_codec_version)
video_codec_profile_list.append(video_codec_profile)
elif track.attrib['type'] == 'Audio':
silence = False
essenceTrackEncod_au = ififuncs.get_metadata(
"ns:Format", track, mediainfo_namespace)
audio_codec_list.append(essenceTrackEncod_au)
#acodec_attributes = get_attributes(track.getparent(), pbcore_namespace)
audio_codecid = ififuncs.get_metadata(
"ns:CodecID", track, mediainfo_namespace)
essenceTrackSampling = ififuncs.get_mediainfo(
'samplerate', '--inform=Audio;%SamplingRate_String%',
source)
sample_rate_list.append(essenceTrackSampling)
essenceBitDepth_au = ififuncs.get_metadata(
"ns:BitDepth", track, mediainfo_namespace)
audio_codecid_list.append(audio_codecid)
au_bitdepth_list.append(essenceBitDepth_au)
channels = ififuncs.get_metadata("//ns:Channels", track,
mediainfo_namespace)
channels_list.append(channels)
if audio_only:
essenceTrackEncodvid = 'n/a'
video_codecid = 'n/a'
video_codec_version = 'n/a'
video_codec_profile = 'n/a'
ScanType = ififuncs.get_metadata("//ns:ScanType", new_root,
mediainfo_namespace)
scan_types.append(ScanType)
matrix_coefficients = ififuncs.get_metadata("//ns:matrix_coefficients",
new_root,
mediainfo_namespace)
timecode_source, starting_timecode = get_timecode(
pbcore_namespace, root, source)
timecode_list.append(starting_timecode)
matrix_list.append(matrix_coefficients)
transfer_characteris = ififuncs.get_metadata(
"//ns:transfer_characteristics", new_root, mediainfo_namespace)
transfer_list.append(transfer_characteris)
colour_primaries = ififuncs.get_metadata("//ns:colour_primaries",
new_root, mediainfo_namespace)
colour_primaries_list.append(colour_primaries)
try:
if audio_only:
FrameCount = 'n/a'
else:
# increment if multiple objects are present
try:
FrameCount += int(
ififuncs.get_metadata("//ns:FrameCount", new_root,
mediainfo_namespace))
except ValueError:
# don't increment if multiple values are returned as str
FrameCount = ififuncs.get_metadata("//ns:FrameCount",
new_root,
mediainfo_namespace)
except TypeError:
# workaround for silent pic in DCP
FrameCount = 'n/a'
instantFileSize_byte += int(
ififuncs.get_metadata("//ns:FileSize", new_root,
mediainfo_namespace))
instantDataRate = round(
float(
ififuncs.get_mediainfo('OverallBitRate',
'--inform=General;%OverallBitRate%',
source)) / 1000 / 1000, 2)
instantTracks = ififuncs.get_number_of_tracks(source)
track_count_list.append(instantTracks)
if stl is True:
track_count_list.append('STL sidecar')
ms += ififuncs.get_milliseconds(source)
ColorSpace = ififuncs.get_metadata("//ns:ColorSpace", new_root,
mediainfo_namespace)
color_spaces.append(ColorSpace)
ChromaSubsampling = get_metadata("//ns:ChromaSubsampling", new_root,
mediainfo_namespace)
chroma.append(ChromaSubsampling)
instantMediaty = get_metadata("//ns:instantiationMediaType", root,
pbcore_namespace)
if audio_only:
essenceFrameSize = 'n/a'
else:
essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
pbcore_namespace)
frame_sizes.append(essenceFrameSize)
PixelAspectRatio = ififuncs.get_metadata("//ns:PixelAspectRatio",
new_root, mediainfo_namespace)
par_list.append(PixelAspectRatio)
general_root = new_root.xpath("//ns:track[@type='General']",
namespaces={'ns':
mediainfo_namespace})[0]
instantiationStandar = ififuncs.get_metadata("ns:Format", general_root,
mediainfo_namespace)
container_list.append(instantiationStandar)
essenceFrameRate = ififuncs.get_metadata("//ns:FrameRate", new_root,
mediainfo_namespace)
fps_list.append(essenceFrameRate)
essenceAspectRatio = ififuncs.get_mediainfo(
'DAR', '--inform=Video;%DisplayAspectRatio_String%', source)
Interlacement = ififuncs.get_metadata("//ns:ScanOrder", new_root,
mediainfo_namespace)
# FFV1/MKV seems to have this scanorder metadata here rather than Interlacement
# FFV1/MKV is the only example I've seen so far that behaves like this :|
# It could be that Interlacement is set at a codec level for FFV1, but others are
# declared at the container level..
if Interlacement == 'n/a':
Interlacement = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ScanOrder']",
root, pbcore_namespace)
interlace_list.append(Interlacement)
Compression_Mode = ififuncs.get_metadata("//ns:Compression_Mode",
new_root, mediainfo_namespace)
colour_range = ififuncs.get_metadata("//ns:colour_range", new_root,
mediainfo_namespace)
# this needs to be clarified as it exists in general and codec
format_version = ififuncs.get_metadata("ns:Format_Version",
general_root,
mediainfo_namespace)
app_company_name = ififuncs.get_metadata(
"//ns:Encoded_Application_CompanyName", new_root,
mediainfo_namespace)
app_name = ififuncs.get_metadata("//ns:Encoded_Application_Name",
new_root, mediainfo_namespace)
app_version = ififuncs.get_metadata("//ns:Encoded_Application_Version",
new_root, mediainfo_namespace)
library_name = ififuncs.get_metadata("//ns:Encoded_Library_Name",
new_root, mediainfo_namespace)
if library_name == 'n/a':
library_name = ififuncs.get_metadata("//ns:Encoded_Library",
general_root,
mediainfo_namespace)
library_version = ififuncs.get_metadata("//ns:Encoded_Library_Version",
new_root, mediainfo_namespace)
compression_list.append(Compression_Mode)
instantiationDate_mo = get_metadata(
"//ns:instantiationDate[@dateType='file modification']", root,
pbcore_namespace)
instantDate_other = 'n/a'
instantDate_type = 'n/a'
pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
pix_fmt_list.append(pix_fmt)
audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
audio_fmt_list.append(audio_fmt)
essenceBitDepth_vid = ififuncs.get_mediainfo(
'duration', '--inform=Video;%BitDepth%', source)
if silence:
audio_codecid = 'n/a'
essenceBitDepth_au = 'n/a'
essenceTrackEncod_au = 'n/a'
essenceTrackSampling = 'n/a'
channels = 'n/a'
'''
video_codecid = vcodec_attributes['ref']
video_codecid_list.append(video_codecid)
try:
video_codec_version = vcodec_attributes['version']
except KeyError:
video_codec_version = 'n/a'
try:
video_codec_profile = vcodec_attributes['annotation'][8:]
except KeyError:
video_codec_profile = 'n/a'
'''
metadata_error = ''
metadata_list = [
scan_types, matrix_list, transfer_list, colour_primaries_list,
color_spaces, chroma, frame_sizes, par_list, container_list, fps_list,
sample_rate_list, track_count_list, interlace_list, compression_list,
pix_fmt_list, audio_fmt_list, audio_codecid_list, audio_codec_list,
au_bitdepth_list, video_codecid_list, video_codec_version_list,
video_codec_profile_list, channels_list, timecode_list
]
for i in metadata_list:
if len(set(i)) > 1:
metadata_error += 'WARNING - Your metadata values are not the same for all files - but this could be a false positive if dealing with atomised audio and video as with DCP: %s\n' % set(
i)
if args.p:
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata mismatch - Your metadata values are not the same for all files - but this could be a false positive if dealing with atomised audio and video as with DCP: %s'
% set(i))
print(metadata_error)
tc = ififuncs.convert_millis(ms)
instantiationDuratio = ififuncs.convert_timecode(25, tc)
if args.donor:
Donor = args.donor
else:
Donor = ''
Edited_By = user
Date_Created = ''
Date_Last_Modified = ''
Film_Or_Tape = 'Digital AV Object'
Date_Of_Donation = ''
if args.reproduction_creator:
reproduction_creator = args.reproduction_creator
else:
reproduction_creator = ''
if args.acquisition_type:
if acquisition_type == 'Reproduction':
Date_Of_Donation = instantiationDate_mo.split('T')[0]
# if a reproduction, then there's no Donor/transfer of title.
Donor = 'n/a'
else:
Date_Of_Donation = args.donation_date
Habitat = ''
backup_habitat = ''
Type_Of_Deposit = acquisition_type
if args.depositor_reference:
Depositor_Reference = args.depositor_reference
else:
Depositor_Reference = ''
Master_Viewing = 'Preservation Object'
Language_Version = ''
Condition_Rating = ''
Companion_Elements = ''
TTape_Origin = args.parent
EditedNew = user
FIO = 'In'
CollectionTitle = ''
Created_By = user
instantTimeStart = 'n/a'
instantFileSize_gigs = round(
float(instantFileSize_byte) / 1024 / 1024 / 1024, 3)
instantColors = 'n/a'
instantLanguage = 'n/a'
instantAltMo = 'n/a'
instantiationChanCon = 'n/a'
'''
no idea why these are here
colour_range = colour_range
format_version = format_version
'''
TimeCode_FirstFrame = process_mixed_values(timecode_list)
pix_fmt = process_mixed_values(pix_fmt_list)
audio_fmt = process_mixed_values(audio_fmt_list)
instantTracks = process_mixed_values(track_count_list)
TimeCode_Source = timecode_source
reproduction_reason = ''
dig_object_descrip = ififuncs.get_digital_object_descriptor(args.input)
if 'STL' in dig_object_descrip:
dig_object_descrip = 'AS-11 package'
dcp_check = ififuncs.find_cpl(args.input)
if dcp_check is not None:
essenceFrameSize, ChromaSubsampling, ColorSpace, FrameCount, essenceAspectRatio, instantiationDuratio, PixelAspectRatio, ScanType, dig_object_descrip, instantTracks, instantDataRate, essenceBitDepth_vid, instantMediaty = check_dcp(
dcp_check)
ififuncs.append_csv(csv_filename, [
Reference_Number,
Donor,
Edited_By,
Date_Created,
Date_Last_Modified,
Film_Or_Tape,
Date_Of_Donation,
Accession_Number,
Habitat,
backup_habitat,
TTape_Origin,
Type_Of_Deposit,
Depositor_Reference,
Master_Viewing,
Language_Version,
Condition_Rating,
Companion_Elements,
EditedNew,
FIO,
CollectionTitle,
Created_By,
instantiationIdentif,
instantDate_other,
instantDate_type,
instantiationDate_mo,
instantiationStandar,
instantMediaty,
instantFileSize_byte,
instantFileSize_gigs,
instantTimeStart,
instantDataRate,
instantTracks,
instantColors,
instantLanguage,
instantAltMo,
essenceTrackEncodvid,
essenceFrameRate,
essenceTrackSampling,
essenceBitDepth_vid,
essenceFrameSize,
essenceAspectRatio,
essenceTrackEncod_au,
essenceBitDepth_au,
instantiationDuratio,
instantiationChanCon,
PixelAspectRatio,
FrameCount,
ColorSpace,
ChromaSubsampling,
ScanType,
Interlacement,
Compression_Mode,
colour_primaries,
transfer_characteris,
matrix_coefficients,
pix_fmt,
audio_fmt,
audio_codecid,
video_codecid,
video_codec_version,
video_codec_profile,
channels,
colour_range,
format_version,
TimeCode_FirstFrame,
TimeCode_Source,
app_company_name,
app_name,
app_version,
library_name,
library_version,
reproduction_creator,
reproduction_reason,
dig_object_descrip,
])
if args.p:
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
% (csv_filename))
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(md5_manifest, csv_filename)
print((' - Updating %s with %s' % (md5_manifest, csv_filename)))
ififuncs.sha512_update(sha512_manifest, csv_filename)
print((' - Updating %s with %s' % (sha512_manifest, csv_filename)))
print(metadata_error)
def main(args_):
# if multiple file are present, this script will treat them as a single
# instantiation/representation and get aggregate metadata about the whole
# package. For now, this will be a clumsy implementation - the first file
# will provide most metadata. Things like duration/bitrate/filesize
# will be calculated as a whole.
# Although another way would be that every call is looped, and if
# this could catch files that should not be in the package, eg. a 4:2:2
# file in a 4:2:0 package..
# yup - do it that way!
args = parse_args(args_)
all_files = ififuncs.recursive_file_list(args.input)
silence = True
if args.user:
user = args.user
else:
user = ififuncs.get_user()
for dirs in os.listdir(args.input):
if ififuncs.validate_uuid4(dirs) is None:
instantiationIdentif = dirs
Accession_Number = get_accession_number(args.input)
Reference_Number = get_reference_number(args.input)
if args.p:
for root, _, filenames in os.walk(args.input):
if os.path.basename(root) == 'metadata':
metadata_dir = root
elif os.path.basename(root) == 'logs':
logs_dir = root
csv_filename = os.path.join(metadata_dir, Accession_Number + '.csv')
sipcreator_log = os.path.join(logs_dir,
instantiationIdentif + '_sip_log.log')
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py started')
ififuncs.generate_log(
sipcreator_log, 'eventDetail=makepbcore.py %s' %
ififuncs.get_script_version('makepbcore.py'))
ififuncs.generate_log(sipcreator_log,
'Command line arguments: %s' % args)
ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
else:
csv_filename = 'blaa.csv'
print ' - Metadata will be stored in %s' % csv_filename
for filenames in os.listdir(args.input):
if '_manifest.md5' in filenames:
md5_manifest = os.path.join(args.input, filenames)
elif 'manifest-sha512.txt' in filenames:
sha512_manifest = os.path.join(args.input, filenames)
make_csv(csv_filename)
ms = 0
FrameCount = 0
instantFileSize = 0
instantFileSize_gigs = 0
for source in all_files:
metadata = subprocess.check_output(
['mediainfo', '--Output=PBCore2', source])
root = etree.fromstring(metadata)
print(' - Analysing %s') % source
pbcore_namespace = root.xpath('namespace-uri(.)')
track_type = root.xpath('//ns:essenceTrackType',
namespaces={'ns': pbcore_namespace})
if len(track_type) > 0:
for track in track_type:
if track.text == 'Video':
essenceTrackEncodvid = get_metadata(
"ns:essenceTrackEncoding", track.getparent(),
pbcore_namespace)
vcodec_attributes = get_attributes(track.getparent(),
pbcore_namespace)
elif track.text == 'Audio':
silence = False
essenceTrackEncod_au = get_metadata(
"ns:essenceTrackEncoding", track.getparent(),
pbcore_namespace)
acodec_attributes = get_attributes(track.getparent(),
pbcore_namespace)
ScanType = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ScanType']", root,
pbcore_namespace)
matrix_coefficients = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='matrix_coefficients']",
root, pbcore_namespace)
transfer_characteris = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='transfer_characteristics']",
root, pbcore_namespace)
colour_primaries = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='color_primaries']",
root, pbcore_namespace)
FrameCount += int(
get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='FrameCount']",
root, pbcore_namespace))
instantFileSize += int(
get_metadata("//ns:instantiationFileSize", root, pbcore_namespace))
instantDataRate = round(
float(
ififuncs.get_mediainfo('OverallBitRate',
'--inform=General;%OverallBitRate%',
source)) / 1000 / 1000, 2)
ms += ififuncs.get_milliseconds(source)
ColorSpace = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ColorSpace']", root,
pbcore_namespace)
ChromaSubsampling = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='ChromaSubsampling']",
root, pbcore_namespace)
instantMediaty = get_metadata("//ns:instantiationMediaType", root,
pbcore_namespace)
essenceFrameSize = get_metadata("//ns:essenceTrackFrameSize", root,
pbcore_namespace)
essenceAspectRatio = get_metadata("//ns:essenceTrackAspectRatio", root,
pbcore_namespace)
PixelAspectRatio = get_metadata(
"//ns:essenceTrackAnnotation[@annotationType='PixelAspectRatio']",
root, pbcore_namespace)
instantiationStandar = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Format']", root,
pbcore_namespace)
essenceFrameRate = get_metadata("//ns:essenceTrackFrameRate", root,
pbcore_namespace)
essenceTrackSampling = get_metadata("//ns:essenceTrackSamplingRate",
root, pbcore_namespace)
Interlacement = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Interlacement']",
root, pbcore_namespace)
Compression_Mode = get_metadata(
"//ns:instantiationAnnotation[@annotationType='Compression_Mode']",
root, pbcore_namespace)
instantiationDate_modified = get_metadata(
"//ns:instantiationDate[@dateType='file modification']", root,
pbcore_namespace)
pix_fmt = ififuncs.get_ffmpeg_fmt(source, 'video')
audio_fmt = ififuncs.get_ffmpeg_fmt(source, 'audio')
if not silence:
audio_codecid = acodec_attributes['ref']
essenceBitDepth_au = ififuncs.get_mediainfo(
'duration', '--inform=Audio;%BitDepth%', source)
else:
audio_codecid = 'n/a'
essenceBitDepth_au = 'n/a'
essenceTrackEncod_au = 'n/a'
video_codecid = vcodec_attributes['ref']
try:
video_codec_version = vcodec_attributes['version']
except KeyError:
video_codec_version = 'n/a'
try:
video_codec_profile = vcodec_attributes['annotation'][8:]
except KeyError:
video_codec_profile = 'n/a'
tc = ififuncs.convert_millis(ms)
instantiationDuratio = ififuncs.convert_timecode(25, tc)
Donor = ''
Edited_By = user
Date_Created = ''
Date_Last_Modified = ''
Film_Or_Tape = 'Digital File'
Date_Of_Donation = ''
Habitat = ''
Type_Of_Deposit = ''
Depositor_Reference = ''
Master_Viewing = 'Preservation Master'
Language_Version = ''
Condition_Rating = ''
Companion_Elements = ''
EditedNew = user
FIO = 'In'
CollectionTitle = ''
Created_By = user
instantiationDimensi = ''
instantiationLocatio = ''
instantTimeStart = ''
instantFileSize_gigs = round(
float(instantFileSize) / 1024 / 1024 / 1024, 3)
instantColors = ''
instantLanguage = ''
instantAltMo = 'n/a'
essenceBitDepth_vid = ififuncs.get_mediainfo('duration',
'--inform=Video;%BitDepth%',
source)
instantiationChanCon = ''
ififuncs.append_csv(csv_filename, [
Reference_Number, Donor, Edited_By, Date_Created, Date_Last_Modified,
Film_Or_Tape, Date_Of_Donation, Accession_Number, Habitat,
Type_Of_Deposit, Depositor_Reference, Master_Viewing, Language_Version,
Condition_Rating, Companion_Elements, EditedNew, FIO, CollectionTitle,
Created_By, instantiationIdentif, instantiationDate_modified,
instantiationDimensi, instantiationStandar, instantiationLocatio,
instantMediaty, instantFileSize, instantFileSize_gigs,
instantTimeStart, instantDataRate, instantColors, instantLanguage,
instantAltMo, essenceTrackEncodvid, essenceFrameRate,
essenceTrackSampling, essenceBitDepth_vid, essenceFrameSize,
essenceAspectRatio, essenceTrackEncod_au, essenceBitDepth_au,
instantiationDuratio, instantiationChanCon, PixelAspectRatio,
FrameCount, ColorSpace, ChromaSubsampling, ScanType, Interlacement,
Compression_Mode, colour_primaries, transfer_characteris,
matrix_coefficients, pix_fmt, audio_fmt, audio_codecid, video_codecid,
video_codec_version, video_codec_profile
])
if args.p:
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=Technical record creation using PBCore, eventOutcome=%s, agentName=makepbcore'
% (csv_filename))
ififuncs.generate_log(sipcreator_log, 'EVENT = makepbcore.py finished')
ififuncs.checksum_replace(md5_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(md5_manifest, csv_filename)
print ' - Updating %s with %s' % (md5_manifest, csv_filename)
ififuncs.sha512_update(sha512_manifest, csv_filename)
print ' - Updating %s with %s' % (sha512_manifest, csv_filename)
def main(args_):
'''
Launches the various functions that will accession a package
'''
args = parse_args(args_)
source = args.input
uuid_directory = ififuncs.check_for_sip([source])
if uuid_directory is not None:
oe_path = os.path.dirname(uuid_directory)
oe_number = os.path.basename(oe_path)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.number:
if args.number[:3] != 'aaa':
print 'First three characters must be \'aaa\' and last four characters must be four digits'
accession_number = ififuncs.get_accession_number()
elif len(args.number[3:]) != 4:
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
elif not args.number[3:].isdigit():
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
else:
accession_number = args.number
else:
accession_number = ififuncs.get_accession_number()
if args.pbcore:
if args.reference:
Reference_Number = args.reference.upper()
else:
Reference_Number = ififuncs.get_reference_number()
if args.acquisition_type:
acquisition_type = ififuncs.get_acquisition_type(
args.acquisition_type)
print acquisition_type
accession_path = os.path.join(os.path.dirname(oe_path),
accession_number)
uuid = os.path.basename(uuid_directory)
new_uuid_path = os.path.join(accession_path, uuid)
logs_dir = os.path.join(new_uuid_path, 'logs')
sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
if args.force:
proceed = 'Y'
else:
proceed = ififuncs.ask_yes_no('Do you want to rename %s with %s' %
(oe_number, accession_number))
if proceed == 'Y':
os.rename(oe_path, accession_path)
if args.register:
register = args.register
else:
register = make_register()
ififuncs.append_csv(register,
(oe_number.upper()[:2] + '-' + oe_number[2:],
accession_number, '', '', '', '', '', ''))
ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py started')
ififuncs.generate_log(
sipcreator_log, 'eventDetail=accession.py %s' %
ififuncs.get_script_version('accession.py'))
ififuncs.generate_log(sipcreator_log,
'Command line arguments: %s' % args)
ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(
sipcreator_log, 'EVENT = eventType=Identifier assignment,'
' eventIdentifierType=accession number, value=%s' %
accession_number)
ififuncs.generate_log(
sipcreator_log, 'EVENT = eventType=accession,'
' eventIdentifierType=accession number, value=%s' %
accession_number)
sip_manifest = os.path.join(accession_path, uuid) + '_manifest.md5'
sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(new_uuid_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
os.remove(sha512_log)
print('Generating Digital Forensics XML')
dfxml = make_dfxml(args, new_uuid_path, uuid)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
# this is inefficient. The script should not have to ask for reference
# number twice if someone wants to insert the filmographic but do not
# want to make the pbcore csv, perhaps because the latter already exists.
if args.csv:
metadata_dir = os.path.join(new_uuid_path, 'metadata')
package_filmographic = os.path.join(
metadata_dir, Reference_Number + '_filmographic.csv')
insert_filmographic(args.csv, Reference_Number,
package_filmographic)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=Filmographic descriptive metadata added to metadata folder, eventOutcome=%s, agentName=accession.py'
% (package_filmographic))
ififuncs.manifest_update(sip_manifest, package_filmographic)
ififuncs.sha512_update(sha512_manifest, package_filmographic)
print('Filmographic descriptive metadata added to metadata folder')
ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py finished')
ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(sip_manifest, dfxml)
ififuncs.sha512_update(sha512_manifest, dfxml)
if args.pbcore:
makepbcore_cmd = [
accession_path, '-p', '-user', user, '-reference',
Reference_Number
]
if args.parent:
makepbcore_cmd.extend(['-parent', args.parent])
if args.acquisition_type:
makepbcore_cmd.extend(
['-acquisition_type', args.acquisition_type])
if args.donor:
makepbcore_cmd.extend(['-donor', args.donor])
if args.donor:
makepbcore_cmd.extend(
['-depositor_reference', args.depositor_reference])
if args.donation_date:
makepbcore_cmd.extend(['-donation_date', args.donation_date])
makepbcore.main(makepbcore_cmd)
else:
print 'not a valid package. The input should include a package that has been through Object Entry'
user = '******'
print 'Hi Brian, Congratulations on becoming a father!!!'
elif user == '2':
user = '******'
print 'Hi Gavin, Have you renewed your subscription to American Cinematographer?'
elif user == '3':
user = '******'
print 'Hi Raelene, Brian Cash must be on paternity leave'
time.sleep(1)
create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root,dirnames,filenames in os.walk(source_directory):
source_directory = root
if not file_check(source_directory) == 'TIFF':
append_csv(csv_report_filename, (source_directory,'EMPTY DIRECTORY - SKIPPED', 'n/a', 'n/a'))
continue
root_dir = os.path.dirname(os.path.dirname(root))
general_log = root_dir + '/logs/image/%s_image_log.log' % os.path.basename(root_dir)
generate_log(general_log, 'Input = %s' % root)
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
source_manifest = root_dir + '/%s_manifest.md5' % relative_path
generate_log(general_log, 'Generating source manifest via md5deep and storing as %s' % source_manifest)
make_manifest(root_dir, root_dir, source_manifest)
info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir,
os.path.basename(filename))
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename))
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0])
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir,
outputfilename)
fmd5 = "%s/%s_source.framemd5" % (metadata_dir,
os.path.basename(filename))
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]).rstrip()
field_order = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%ScanType%", filename
]).rstrip()
height = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full', "--Inform=Video;%Height%",
filename
]).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i',
filename,
'-c:v',
'ffv1', # Use FFv1 codec
'-g',
'1', # Use intra-frame only aka ALL-I aka GOP=1
'-level',
'3', # Use Version 3 of FFv1
'-c:a',
'copy', # Copy and paste audio bitsream with no transcoding
'-map',
'0',
'-dn',
'-report',
'-slicecrc',
'1',
'-slices',
'16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += ['-vf', 'setfield=tff, setdar=4/3']
ffv1_command += [
output,
'-f',
'framemd5',
'-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation completed.')
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.')
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
pix_fmt = get_ffmpeg_fmt(filename, 'video')
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i',
output,
'-report',
'-pix_fmt',
pix_fmt,
'-f',
'framemd5',
'-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file completed')
source_video_size = get_mediainfo('source_video_size',
"--inform=General;%FileSize%",
filename)
ffv1_video_size = get_mediainfo('ffv1_video_size',
'--inform=General;%FileSize%', output)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
try:
shutil.copy(sys.argv[0], log_dir)
except IOError:
pass
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename,
(output, 'LOSSLESS', source_video_size, ffv1_video_size,
compression_ratio))
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(output, 'LOSSLESS - different PAR', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(csv_report_filename,
(output, 'NOT LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
def main(args_):
'''
Launches the various functions that will accession a package
'''
args = parse_args(args_)
input = args.input
uuid_directory = ififuncs.check_for_sip([input])
if uuid_directory is not None:
oe_path = os.path.dirname(uuid_directory)
oe_number = os.path.basename(oe_path)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.number:
if args.number[:3] != 'aaa':
print 'First three characters must be \'aaa\' and last four characters must be four digits'
accession_number = ififuncs.get_accession_number()
elif len(args.number[3:]) != 4:
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
elif not args.number[3:].isdigit():
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
else:
accession_number = args.number
else:
accession_number = ififuncs.get_accession_number()
accession_path = os.path.join(
os.path.dirname(oe_path), accession_number
)
uuid = os.path.basename(uuid_directory)
new_uuid_path = os.path.join(accession_path, uuid)
logs_dir = os.path.join(new_uuid_path, 'logs')
sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
if args.force:
proceed = 'Y'
else:
proceed = ififuncs.ask_yes_no(
'Do you want to rename %s with %s' % (oe_number, accession_number)
)
if proceed == 'Y':
os.rename(oe_path, accession_path)
register = make_register()
ififuncs.append_csv(register, (oe_number.upper()[:2] + '-' + oe_number[2:6], accession_number, '','','','', ''))
ififuncs.generate_log(
sipcreator_log,
'EVENT = accession.py started'
)
ififuncs.generate_log(
sipcreator_log,
'eventDetail=accession.py %s' % ififuncs.get_script_version('accession.py')
)
ififuncs.generate_log(
sipcreator_log,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = agentName=%s' % user
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = eventType=Identifier assignment,'
' eventIdentifierType=accession number, value=%s'
% accession_number
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = eventType=accession,'
' eventIdentifierType=accession number, value=%s'
% accession_number
)
sip_manifest = os.path.join(
accession_path, uuid
) + '_manifest.md5'
sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
sha512_manifest = os.path.join(
os.path.dirname(new_uuid_path), uuid + '_manifest-sha512.txt'
)
ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
os.remove(sha512_log)
dfxml = make_dfxml(args, new_uuid_path, uuid)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = accession.py finished'
)
ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(sip_manifest, dfxml)
ififuncs.sha512_update(sha512_manifest, dfxml)
if args.pbcore:
makepbcore.main([accession_path, '-p', '-user', user])
else:
print 'not a valid package. The input should include a package that has been through Object Entry'
def main(args_):
'''
Retrospectively updates older FFV1/DV packages in order to meet our current
packaging requirements. This should allow accession.py and makepbcore.py to run as
expected. This script should work on files created by:
makeffv1.py
dvsip.py
loopline.py
'''
args = parse_args(args_)
user = ififuncs.get_user()
new_object_entry = get_numbers(args)
filmographic_csv = args.filmographic
technical_csv = args.technical
filmographic_oe_list = []
filmo_csv_extraction = ififuncs.extract_metadata(filmographic_csv)
tech_csv_extraction = ififuncs.extract_metadata(technical_csv)
register = make_register()
for line_item in filmo_csv_extraction[0]:
dictionary = {}
oe_number = line_item['Object Entry'].lower()
dictionary['title'] = line_item['Title']
if dictionary['title'] == '':
dictionary['title'] = '%s - %s' % (line_item['TitleSeries'],
line_item['EpisodeNo'])
dictionary['uppercase_dashed_oe'] = oe_number.upper()
for tech_record in tech_csv_extraction[0]:
if tech_record['Reference Number'] == dictionary[
'uppercase_dashed_oe']:
dictionary['source_accession_number'] = tech_record[
'Accession Number']
dictionary['filmographic_reference_number'] = tech_record[
'new_ref']
# this transforms OE-#### to oe####
dictionary['old_oe'] = oe_number[:2] + oe_number[3:]
filmographic_oe_list.append(dictionary)
for oe_package in filmographic_oe_list:
for root, _, filenames in os.walk(args.input):
if os.path.basename(root) == oe_package['old_oe']:
old_oe_path = root
old_oe = os.path.basename(root)
log_dir = os.path.join(root, 'logs')
for files in os.listdir(log_dir):
if '.mov_log.log' in files:
log = os.path.join(log_dir, files)
manifest = os.path.join(os.path.dirname(root),
old_oe + '_manifest.md5')
uuid = ififuncs.create_uuid()
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
ififuncs.generate_log(log,
'EVENT = loopline_repackage.py started')
ififuncs.generate_log(
log, 'eventDetail=loopline_repackage.py %s' %
ififuncs.get_script_version('loopline_repackage.py'))
ififuncs.generate_log(log, 'Command line arguments: %s' % args)
ififuncs.generate_log(log, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(log, uuid_event)
ififuncs.generate_log(
log, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' %
new_object_entry)
ififuncs.generate_log(
log, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=Filmographic reference number , value=%s'
% oe_package['filmographic_reference_number'])
oe_package['new_object_entry'] = new_object_entry
print('Transforming %s into %s' %
(oe_package['old_oe'], oe_package['new_object_entry']))
ififuncs.generate_log(
log, 'Relationship, derivation, has source=%s' %
oe_package['source_accession_number'])
old_uuid_path = os.path.join(os.path.dirname(root), uuid)
new_oe_path, new_uuid_path = move_files(
root, new_object_entry, old_oe_path, old_uuid_path, uuid)
updated_lines = update_manifest(manifest, old_oe, uuid)
new_manifest = os.path.join(new_oe_path,
uuid) + '_manifest.md5'
shutil.move(manifest, new_manifest)
with open(new_manifest, 'w') as fo:
for lines in updated_lines:
fo.write(lines)
new_logs_path = os.path.join(new_uuid_path, 'logs')
for files in os.listdir(new_logs_path):
if '.mov_log.log' in files:
log = os.path.join(new_logs_path, files)
logname = rename_files(new_uuid_path, old_oe, uuid,
new_manifest, log)
date_modified, extension = get_date_modified(new_uuid_path)
# This normally would be bad practise, but this project only has two formats. MOV/DV and FFv1/MKV
if extension == '.mkv':
av_format = 'FFV1/PCM/Matroska'
elif extension == '.mov':
av_format = 'DV/PCM/QuickTime'
provenance_string = 'Reproduction of %s' % oe_package[
'source_accession_number']
ififuncs.append_csv(
register,
(oe_package['new_object_entry'].upper()[:2] + '-' +
oe_package['new_object_entry'][2:], date_modified, '1',
av_format, oe_package['title'], 'contact_name',
'Reproduction', '', provenance_string, '', ''))
ififuncs.generate_log(
logname, 'EVENT = loopline_repackage.py finished')
ififuncs.checksum_replace(new_manifest, logname, 'md5')
oe_digits = int(os.path.basename(new_oe_path)[2:]) + 1
new_object_entry = 'oe' + str(oe_digits)
os.path.abspath(dpx_filename), '-strict', '-2', '-c:v', 'ffv1',
'-level', '3', '-pix_fmt', 'rgb48le',
output_dirname + '/video/' + os.path.basename(root) + '.mkv'
]
print ffv12dpx
subprocess.call(ffv12dpx, env=env_dict)
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(
output_dirname) + '/' + parent_basename + '_manifest.md5'
ffv1_path = output_dirname + '/video/' + os.path.basename(root) + '.mkv'
ffv1_md5 = output_dirname + '/md5/' + os.path.basename(
root) + 'ffv1.framemd5'
subprocess.call([
'ffmpeg', '-i', ffv1_path, '-pix_fmt', 'rgb48le', '-f', 'framemd5',
ffv1_md5
])
#other_textfile = other[1]
judgement = diff_textfiles(source_textfile, ffv1_md5)
make_manifest(output_parent_directory, os.path.basename(output_dirname),
manifest_textfile)
finish = datetime.datetime.now()
print total_size
ffv1_size = os.path.getsize(ffv1_path)
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
append_csv(csv_report_filename, (parent_basename, judgement, start, finish,
total_size, ffv1_size, comp_ratio))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
source_directory = root + '/tiff_scans'
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
source_manifest = source_parent_dir + '/%s_manifest.md5' % relative_path
make_manifest(os.path.dirname(source_directory), os.path.basename(source_directory), source_manifest)
info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
output_dirname = info[0]
source_textfile = info[1]
image_seq_without_container = info[2]
tiff_filename = image_seq_without_container + "%06d.tiff"
dpx_filename = image_seq_without_container + "%06d.dpx"
logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
env_dict = set_environment(logfile)
tiff2dpx = ['ffmpegnometadata','-report','-f','image2','-framerate','24', '-i', tiff_filename ,output_dirname + '/image/dpx_files' '/' + dpx_filename]
print tiff2dpx
subprocess.call(tiff2dpx,env=env_dict)
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5'
other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx', 'dpx_framemd5')
other_textfile = other[1]
judgement = diff_textfiles(source_textfile, other_textfile)
make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
finish = datetime.datetime.now()
append_csv(csv_report_filename, (parent_basename,judgement, start, finish))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (
metadata_dir, os.path.basename(filename)
)
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename)
)
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0]
)
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
fmd5 = "%s/%s_source.framemd5" % (
metadata_dir, os.path.basename(filename)
)
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output(
[
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]
).rstrip()
field_order = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%ScanType%", filename
]
).rstrip()
height = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%Height%",
filename
]
).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i', filename,
'-c:v', 'ffv1', # Use FFv1 codec
'-g', '1', # Use intra-frame only aka ALL-I aka GOP=1
'-level', '3', # Use Version 3 of FFv1
'-c:a', 'copy', # Copy and paste audio bitsream with no transcoding
'-map', '0',
'-dn',
'-report',
'-slicecrc', '1',
'-slices', '16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += [
'-vf',
'setfield=tff, setdar=4/3'
]
ffv1_command += [
output,
'-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation completed.'
)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.'
)
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', output,
'-report',
'-f', 'framemd5', '-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log,
'makeffv1.py Framemd5 generation of output file completed'
)
source_video_size = get_mediainfo(
'source_video_size', "--inform=General;%FileSize%", filename
)
ffv1_video_size = get_mediainfo(
'ffv1_video_size', '--inform=General;%FileSize%', output
)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)
):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(
csv_report_filename, (
output,
'LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(
output,
'LOSSLESS - different PAR',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(
csv_report_filename,
(
output, 'NOT LOSSLESS',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
user = '******'
print 'Hi Brian, Congratulations on becoming a father!!!'
elif user == '2':
user = '******'
print 'Hi Gavin, Have you renewed your subscription to American Cinematographer?'
elif user == '3':
user = '******'
print 'Hi Raelene, Brian must be out of the office'
time.sleep(1)
create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root,dirnames,filenames in os.walk(source_directory):
source_directory = root
if not file_check(source_directory) == 'TIFF':
append_csv(csv_report_filename, (source_directory,'EMPTY DIRECTORY - SKIPPED', 'n/a', 'n/a'))
continue
root_dir = os.path.dirname(os.path.dirname(root))
general_log = root_dir + '/logs/image/%s_image_log.log' % os.path.basename(root_dir)
generate_log(general_log, 'Input = %s' % root)
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
source_manifest = root_dir + '/%s_manifest.md5' % relative_path
generate_log(general_log, 'Generating source manifest via md5deep and storing as %s' % source_manifest)
make_manifest(root_dir, root_dir, source_manifest)
info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
os.path.basename(source_directory), source_manifest)
info = make_framemd5(source_directory, 'tiff', 'tiff_framemd5')
output_dirname = info[0]
source_textfile = info[1]
image_seq_without_container = info[2]
tiff_filename = image_seq_without_container + "%06d.tiff"
dpx_filename = image_seq_without_container + "%06d.dpx"
logfile = output_dirname + '/image/logs/%sdpx_transcode.log' % image_seq_without_container
env_dict = set_environment(logfile)
tiff2dpx = [
'ffmpegnometadata', '-report', '-f', 'image2', '-framerate', '24',
'-i', tiff_filename, output_dirname + '/image/dpx_files'
'/' + dpx_filename
]
print tiff2dpx
subprocess.call(tiff2dpx, env=env_dict)
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(
output_dirname) + '/' + parent_basename + '_manifest.md5'
other = make_framemd5(output_dirname + '/image/dpx_files', 'dpx',
'dpx_framemd5')
other_textfile = other[1]
judgement = diff_textfiles(source_textfile, other_textfile)
make_manifest(output_parent_directory,
os.path.basename(output_dirname), manifest_textfile)
finish = datetime.datetime.now()
append_csv(csv_report_filename,
(parent_basename, judgement, start, finish))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
def representation_uuid_csv(filmographic, source_accession, uuid):
uuid_csv = os.path.expanduser('~/Desktop/uuid.csv')
print os.path.abspath(uuid_csv)
if not os.path.isfile(uuid_csv):
create_csv(uuid_csv, ('reference number','source accession number' 'uuid'))
append_csv(uuid_csv, (filmographic, source_accession, uuid) )
with open(source_textfile) as f1:
with open(ffv1_md5) as f2:
for (lineno1,
line1), (lineno2,
line2) in itertools.izip(read_non_comment_lines(f1),
read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename,
(parent_basename, judgement, start, finish, transcode_start,
transcode_finish, transcode_time, sequence_length, fps,
total_size, ffv1_size, pix_fmt, container, width, height,
comp_ratio))
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
append_csv(
csv_report_filename,
(parent_basename, 'LOSSLESS - different PAR', start, finish,
transcode_start, transcode_finish, transcode_time,
sequence_length, fps, total_size, ffv1_size, pix_fmt,
container, width, height, comp_ratio))
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
print csv_report_filename
def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
while permission not in ('Y', 'y', 'N', 'n'):
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission == 'y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime(
"_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename,
('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root, dirnames, filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root # + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root, files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(
os.path.dirname(os.path.dirname(master_parent_dir)) +
'/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(
os.path.dirname(
master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(
os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename(
master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_manifest.md5'
audio_dir_list = os.listdir(master_parent_dir + '/objects/audio')
for audio_files in audio_dir_list:
if not audio_files[0] == '.':
if audio_files.endswith('.wav'):
master_audio = master_parent_dir + '/objects/audio/' + audio_files
audio_date_modified = get_date_modified(master_audio)
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
image_date_modified = info[3]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
intellectual_entity_uuid = str(uuid.uuid4())
source_representation_uuid = premis_description(
master_object_dir, master_parent_dir + '/objects/audio', user,
image_date_modified, audio_date_modified,
intellectual_entity_uuid)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir, audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(
mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores = [
'ffmpeg', '-y', '-f', 'image2', '-framerate', '24',
'-start_number', start_number, '-i', root + '/' + dpx_filename,
'-i', audio_file, '-c:v', 'prores', '-profile:v', '3', '-c:a',
'pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov',
'-f', 'framemd5', '-an', master_metadata_dir + '/image/' +
os.path.basename(master_parent_dir) + '.framemd5', '-c:a',
'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir +
'/audio/' + os.path.basename(master_parent_dir) + '.framemd5'
]
print seq2prores
subprocess.call(seq2prores, env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(
mezzanine_file)
items = {
"workflow": "seq2prores",
"oe": 'n/a',
"filmographic": split_list[0],
"sourceAccession": split_list[1],
"interventions": ['placeholder'],
"prepList": ['placeholder'],
"user": user
}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis,
premis_namespace, premisxml,
representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(
xml_info[2]
) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_intellectual_entity(premisxml, premis_namespace, doc,
premis, items, intellectual_entity_uuid)
create_representation(premisxml, premis_namespace, doc, premis,
items, linking_representation_uuids,
representation_uuid, sequence,
intellectual_entity_uuid)
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineMachineAgent = make_agent(
premis, macMiniTelecineMachineAgent_events,
'230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineOSAgent = make_agent(
premis, macMiniTelecineMachineOSAgent_events,
'9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid]
hashlibAgent = make_agent(premis, hashlib_events,
'9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid]
ffmpegAgent = make_agent(premis, ffmpegAgent_events,
'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid, prores_event_uuid]
operatorAgent = make_agent(premis, operatorEvents, user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(
premis, 'creation',
'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio',
[
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
ffmpegAgent, operatorAgent
], prores_event_uuid, [representation_uuid], 'outcome', 'now')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
tracexml = "%s/%s_mediatrace.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
audio_mediatracexml = "%s/%s_mediatrace.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml, 'audiomediaxmlinput',
master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml, 'audiomediatraceinput',
master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(
mezzanine_mediainfoxml, 'mediaxmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(
tracexml, 'mediatracexmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest,
master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest,
mezzanine_parent_dir)
make_event(premis, 'message digest calculation',
'Checksum manifest for whole package created', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
operatorAgent
], final_sip_manifest_uuid, [representation_uuid],
'source', 'now')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename,
(os.path.basename(master_parent_dir), start, finish))
'''
def representation_uuid_csv(filmographic, source_accession, uuid):
uuid_csv = os.path.expanduser('~/Desktop/uuid.csv')
if not os.path.isfile(uuid_csv):
create_csv(uuid_csv, ('reference number', 'source accession number'
'uuid'))
append_csv(uuid_csv, (filmographic, source_accession, uuid))
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
clairmeta_version = clairmeta.__version__
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print(args)
user = ififuncs.determine_user(args)
object_entry = get_object_entry(args)
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
uuid, uuid_event = determine_uuid(args, sip_path)
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
if args.d:
content_title = create_content_title_text(sip_path, args)
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
if args.accession:
accession_number = ififuncs.get_accession_number()
reference_number = ififuncs.get_reference_number()
parent = ififuncs.ask_question(
'What is the parent record? eg MV 1234. Enter n/a if this is a born digital acquisition with no parent.'
)
donor = ififuncs.ask_question(
'Who is the source of acquisition, as appears on the donor agreement? This will not affect Reproductions.'
)
depositor_reference = ififuncs.ask_question(
'What is the donor/depositor number? This will not affect Reproductions.'
)
acquisition_type = ififuncs.get_acquisition_type('')
donation_date = ififuncs.ask_question(
'When was the donation date in DD/MM/YYYY format? Eg. 31/12/1999 - Unfortunately this is NOT using ISO 8601.'
)
if args.zip:
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
inputs[0], args.o, uuid, new_log_textfile)
if args.manifest:
shutil.copy(
args.manifest,
args.manifest.replace('_manifest.md5', '_manifest-md5.txt'))
source_manifest = args.manifest.replace('_manifest.md5',
'_manifest-md5.txt')
else:
source_manifest = os.path.join(
args.o,
os.path.basename(args.i[0]) + '_manifest-md5.txt')
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.hashlib_manifest(args.i[0], source_manifest,
os.path.dirname(args.i[0]))
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
% inputs[0])
makezip_judgement, zip_file = makezip.main([
'-i', inputs[0], '-o',
os.path.join(sip_path, 'objects'), '-basename', uuid + '.zip'
])
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, eventDetail=Source object packed into=%s'
% zip_file)
if makezip_judgement is None:
judgement = 'lossless'
else:
judgement = makezip_judgement
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
else:
log_names = move_files(inputs, sip_path, args, user)
ififuncs.get_technical_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
if args.zip:
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=started, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
ififuncs.manifest_update(new_manifest_textfile, zip_file)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=finished, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.zip:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', [inputxml, inputtracexml, dfxml, source_manifest], '-user',
user, '-new_folder', supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
if 'log_names' in locals():
log_report(log_names)
finish = datetime.datetime.now()
print('\n- %s ran this script at %s and it finished at %s' %
(user, start, finish))
if args.d:
process_dcp(sip_path, content_title, args, new_manifest_textfile,
new_log_textfile, metadata_dir, clairmeta_version)
if args.accession:
register = accession.make_register()
filmographic_dict = ififuncs.extract_metadata(args.csv)[0]
for filmographic_record in filmographic_dict:
if filmographic_record['Reference Number'].lower(
) == reference_number.lower():
if filmographic_record['Title'] == '':
title = filmographic_record[
'TitleSeries'] + '; ' + filmographic_record['EpisodeNo']
else:
title = filmographic_record['Title']
oe_register = make_oe_register()
ififuncs.append_csv(
oe_register,
(object_entry.upper()[:2] + '-' + object_entry[2:], donation_date,
'1', '', title, donor, acquisition_type[1], accession_number,
'Representation of %s|Reproduction of %s' %
(reference_number, parent), ''))
accession_cmd = [
os.path.dirname(sip_path), '-user', user, '-f', '-number',
accession_number, '-reference', reference_number, '-register',
register, '-csv', args.csv, '-pbcore'
]
if not parent.lower() == 'n/a':
accession_cmd.extend(['-parent', parent])
accession_cmd.extend(['-donor', donor])
accession_cmd.extend(['-depositor_reference', depositor_reference])
accession_cmd.extend(['-acquisition_type', acquisition_type[2]])
accession_cmd.extend(['-donation_date', donation_date])
print(accession_cmd)
accession.main(accession_cmd)
return new_log_textfile, new_manifest_textfile
total_size += os.path.getsize(os.path.join(root,files))
output_dirname = info[0]
source_textfile = info[1]
image_seq_without_container = info[2]
start_number = info[3]
container = info[4]
dpx_filename = image_seq_without_container + "%06d." + container
logfile = output_dirname + '/logs/%s_dpx_transcode.log' % os.path.basename(root)
env_dict = set_environment(logfile)
ffv12dpx = ['ffmpeg','-report','-f','image2','-framerate','24', '-start_number', start_number, '-i', os.path.abspath(dpx_filename) ,'-strict', '-2','-c:v','ffv1','-level', '3', '-pix_fmt', 'rgb48le',output_dirname + '/video/' + os.path.basename(root) + '.mkv']
print ffv12dpx
subprocess.call(ffv12dpx,env=env_dict)
parent_basename = os.path.basename(output_dirname)
manifest_textfile = os.path.dirname(output_dirname) + '/' + parent_basename + '_manifest.md5'
ffv1_path = output_dirname + '/video/' + os.path.basename(root) + '.mkv'
ffv1_md5 = output_dirname + '/md5/' + os.path.basename(root) + 'ffv1.framemd5'
subprocess.call(['ffmpeg','-i', ffv1_path, '-pix_fmt', 'rgb48le','-f', 'framemd5', ffv1_md5])
#other_textfile = other[1]
judgement = diff_textfiles(source_textfile, ffv1_md5)
make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
finish = datetime.datetime.now()
print total_size
ffv1_size = os.path.getsize(ffv1_path)
comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
append_csv(csv_report_filename, (parent_basename,judgement, start, finish,total_size, ffv1_size, comp_ratio))
#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
