def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
while permission not in ('Y', 'y', 'N', 'n'):
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission == 'y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime(
"_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename,
('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root, dirnames, filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root # + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root, files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(
os.path.dirname(os.path.dirname(master_parent_dir)) +
'/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(
os.path.dirname(
master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(
os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename(
master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_manifest.md5'
audio_dir_list = os.listdir(master_parent_dir + '/objects/audio')
for audio_files in audio_dir_list:
if not audio_files[0] == '.':
if audio_files.endswith('.wav'):
master_audio = master_parent_dir + '/objects/audio/' + audio_files
audio_date_modified = get_date_modified(master_audio)
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
image_date_modified = info[3]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
intellectual_entity_uuid = str(uuid.uuid4())
source_representation_uuid = premis_description(
master_object_dir, master_parent_dir + '/objects/audio', user,
image_date_modified, audio_date_modified,
intellectual_entity_uuid)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir, audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(
mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores = [
'ffmpeg', '-y', '-f', 'image2', '-framerate', '24',
'-start_number', start_number, '-i', root + '/' + dpx_filename,
'-i', audio_file, '-c:v', 'prores', '-profile:v', '3', '-c:a',
'pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov',
'-f', 'framemd5', '-an', master_metadata_dir + '/image/' +
os.path.basename(master_parent_dir) + '.framemd5', '-c:a',
'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir +
'/audio/' + os.path.basename(master_parent_dir) + '.framemd5'
]
print seq2prores
subprocess.call(seq2prores, env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(
mezzanine_file)
items = {
"workflow": "seq2prores",
"oe": 'n/a',
"filmographic": split_list[0],
"sourceAccession": split_list[1],
"interventions": ['placeholder'],
"prepList": ['placeholder'],
"user": user
}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis,
premis_namespace, premisxml,
representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(
xml_info[2]
) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_intellectual_entity(premisxml, premis_namespace, doc,
premis, items, intellectual_entity_uuid)
create_representation(premisxml, premis_namespace, doc, premis,
items, linking_representation_uuids,
representation_uuid, sequence,
intellectual_entity_uuid)
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineMachineAgent = make_agent(
premis, macMiniTelecineMachineAgent_events,
'230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineOSAgent = make_agent(
premis, macMiniTelecineMachineOSAgent_events,
'9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid]
hashlibAgent = make_agent(premis, hashlib_events,
'9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid]
ffmpegAgent = make_agent(premis, ffmpegAgent_events,
'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid, prores_event_uuid]
operatorAgent = make_agent(premis, operatorEvents, user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(
premis, 'creation',
'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio',
[
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
ffmpegAgent, operatorAgent
], prores_event_uuid, [representation_uuid], 'outcome', 'now')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
tracexml = "%s/%s_mediatrace.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
audio_mediatracexml = "%s/%s_mediatrace.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml, 'audiomediaxmlinput',
master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml, 'audiomediatraceinput',
master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(
mezzanine_mediainfoxml, 'mediaxmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(
tracexml, 'mediatracexmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest,
master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest,
mezzanine_parent_dir)
make_event(premis, 'message digest calculation',
'Checksum manifest for whole package created', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
operatorAgent
], final_sip_manifest_uuid, [representation_uuid],
'source', 'now')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename,
(os.path.basename(master_parent_dir), start, finish))
'''
def premis_description(root_dir, aeo_raw_extract_wav_dir, user,
image_date_modified, audio_date_modified,
intellectual_entity_uuid):
source_directory = root_dir
representation_uuid = str(uuid.uuid4())
premisxml, premis_namespace, doc, premis = setup_xml(source_directory)
split_list = os.path.basename(os.path.dirname(
os.path.dirname(root_dir))).split('_')
if 'ifard2016' in split_list[2]:
split_list[2] = split_list[2].replace('ifard2016', 'IFA-(RD)2016-')
elif 'ifard2017' in split_list[2]:
split_list[2] = split_list[2].replace('ifard2017', 'IFA-(RD)2017-')
audio_items = {
"workflow": "treated audio",
"oe": split_list[0],
"filmographic": split_list[1],
"sourceAccession": split_list[2],
"interventions": ['placeholder'],
"prepList": ['placeholder'],
"user": '******'
}
image_items = {
"workflow": "grade",
"oe": split_list[0],
"filmographic": split_list[1],
"sourceAccession": split_list[2],
"interventions": ['placeholder'],
"prepList": ['placeholder'],
"user": '******'
}
linking_representation_uuids = []
xml_info = make_premis(aeo_raw_extract_wav_dir, audio_items, premis,
premis_namespace, premisxml, representation_uuid,
'nosequence')
linking_representation_uuids.append(xml_info[2])
xml_info = make_premis(source_directory, image_items, premis,
premis_namespace, premisxml, representation_uuid,
'sequence')
print xml_info
linking_representation_uuids.append(xml_info[4])
linking_representation_uuids.append(image_items['sourceAccession'])
create_intellectual_entity(premisxml, premis_namespace, doc, premis,
audio_items, intellectual_entity_uuid)
create_representation(premisxml, premis_namespace, doc, premis,
audio_items, linking_representation_uuids,
representation_uuid, 'sequence',
intellectual_entity_uuid)
doc = xml_info[0]
premisxml = xml_info[1]
premis = doc.getroot()
'''
events:
audio - audio cleaning in rx5
export from protools
image:
crop in avid
grade in baselight
export from avid
framemd5 audio
framemd5 image
whole md5
'''
audio_rx5_uuid = str(uuid.uuid4())
audio_protools_uuid = str(uuid.uuid4())
image_avid_crop_uuid = str(uuid.uuid4())
image_baselight_grade_uuid = str(uuid.uuid4())
package_manifest_uuid = str(uuid.uuid4())
image_framemd5_uuid = str(uuid.uuid4())
ffmpegAgent_events = [image_framemd5_uuid]
hashlib_events = [package_manifest_uuid]
avid_events = [image_avid_crop_uuid, image_baselight_grade_uuid]
protools_events = [audio_protools_uuid]
baselight_events = [image_baselight_grade_uuid]
rx5_events = [audio_rx5_uuid]
macMiniTelecineMachineAgent_events = [
audio_rx5_uuid, package_manifest_uuid, image_framemd5_uuid,
audio_protools_uuid
]
macMiniTelecineMachineOSAgent_events = [
audio_rx5_uuid, package_manifest_uuid, image_framemd5_uuid,
audio_protools_uuid
]
macProTelecineMachineOSAgent_events = [
image_avid_crop_uuid, image_baselight_grade_uuid
]
macProTelecineMachineAgent_events = [
image_avid_crop_uuid, image_baselight_grade_uuid
]
gavin_events = [image_avid_crop_uuid, image_baselight_grade_uuid]
brian_events = [audio_rx5_uuid, audio_protools_uuid]
if user == 'Gavin Martin':
gavin_events += package_manifest_uuid, image_framemd5_uuid
elif user == 'Brian Cash':
brian_events += package_manifest_uuid, image_framemd5_uuid
else:
script_user_events = [
package_manifest_uuid,
image_framemd5_uuid,
]
script_user_Agent = make_agent(premis, script_user_events, user)
gavinAgent = make_agent(premis, gavin_events,
'9cab0b9c-4787-4482-8927-a045178c8e39')
if user == 'Gavin Martin':
script_user_Agent = gavinAgent
brianAgent = make_agent(premis, brian_events,
'0b96a20d-49f5-46e9-950d-4e11242a487e')
if user == 'Brian Cash':
script_user_Agent = brianAgent
macMiniTelecineMachineAgent = make_agent(
premis, macMiniTelecineMachineAgent_events,
'230d72da-07e7-4a79-96ca-998b9f7a3e41')
macProTelecineMachineAgent = make_agent(
premis, macMiniTelecineMachineAgent_events,
'838a1a1b-7ddd-4846-ae8e-3b5ecb4aae55')
macMiniTelecineOSAgent = make_agent(
premis, macMiniTelecineMachineOSAgent_events,
'68f56ede-a1cf-48aa-b1d8-dc9850d5bfcc')
macProTelecineOSAgent = make_agent(premis,
macProTelecineMachineOSAgent_events,
'52adf876-bf30-431c-b0c6-80cc4fd9406c')
ffmpegAgent = make_agent(premis, ffmpegAgent_events,
'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
hashlibAgent = make_agent(premis, hashlib_events,
'9430725d-7523-4071-9063-e8a6ac4f84c4')
avidAgent = make_agent(premis, avid_events,
'11e157a3-1aa7-4195-b816-009a3d47148c')
protoolsAgent = make_agent(premis, protools_events,
'55003bbd-49a4-4c7b-8da2-0d5b9bf10168')
baselightAgent = make_agent(premis, baselight_events,
'8c02d962-5ac5-4e51-a30c-002553134320')
rx5Agent = make_agent(premis, rx5_events,
'e5872957-8ee8-4c20-bd8e-d76e1de01b34')
make_event(premis, 'creation', 'Audio cleanup', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent, rx5Agent,
brianAgent
], audio_rx5_uuid, [linking_representation_uuids[0]], 'outcome',
audio_date_modified)
make_event(premis, 'creation', 'Audio trimming and export', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent, protoolsAgent,
brianAgent
], audio_protools_uuid, [linking_representation_uuids[0]], 'outcome',
audio_date_modified)
make_event(premis, 'creation', 'Import to Avid and remove overscan', [
macProTelecineMachineAgent, macProTelecineOSAgent, avidAgent,
gavinAgent
], image_avid_crop_uuid, xml_info[4], 'outcome', image_date_modified)
make_event(premis, 'creation', 'Colour Correction', [
macProTelecineMachineAgent, macProTelecineOSAgent, baselightAgent,
gavinAgent
], image_baselight_grade_uuid, xml_info[4], 'outcome', image_date_modified)
make_event(premis, 'message digest calculation',
'Frame level checksums of image', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
ffmpegAgent, script_user_Agent
], image_framemd5_uuid, xml_info[4], 'source', 'now')
make_event(premis, 'message digest calculation',
'Checksum manifest for whole package created', [
hashlibAgent, macMiniTelecineMachineAgent,
macMiniTelecineOSAgent, script_user_Agent
], package_manifest_uuid, [representation_uuid], 'source',
'now')
write_premis(doc, premisxml)
return representation_uuid
def premis_description(root_dir, process_counter, total_process, aeo_raw_extract_wav_dir, user, aeolight_workstation):
source_directory = root_dir + '/objects/image'
print 'Process %d of %d - Generating PREMIS XML file' % (process_counter,total_process)
process_counter += 1
representation_uuid = str(uuid.uuid4())
premisxml, premis_namespace, doc, premis = setup_xml(source_directory)
split_list = os.path.basename(root_dir).split('_')
audio_items = {"workflow":"raw audio","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'******'}
image_items = {"workflow":"scanning","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
linking_representation_uuids = []
xml_info = make_premis(aeo_raw_extract_wav_dir, audio_items, premis, premis_namespace, premisxml, representation_uuid, 'nosequence')
linking_representation_uuids.append(xml_info[2])
xml_info = make_premis(source_directory, image_items, premis, premis_namespace,premisxml, representation_uuid, 'sequence')
image_uuids = xml_info[4]
linking_representation_uuids.append(image_uuids)
linking_representation_uuids.append(image_items['sourceAccession'])
audio_file_uuid = linking_representation_uuids[0]
create_representation(premisxml, premis_namespace, doc, premis, audio_items,linking_representation_uuids, representation_uuid, 'sequence' )
doc = xml_info[0]
premisxml = xml_info[1]
premis = doc.getroot()
extract_uuid = str(uuid.uuid4())
capture_received_uuid = str(uuid.uuid4())
audio_premis_checksum_uuid = str(uuid.uuid4())
audio_framemd5_uuid = str(uuid.uuid4())
scanning_uuid = str(uuid.uuid4())
premis_checksum_uuid = str(uuid.uuid4())
package_manifest_uuid = str(uuid.uuid4())
aeolight_events = [extract_uuid]
aeolightAgent = make_agent(premis,aeolight_events, 'bc3de900-3903-4764-ab91-2ce89977d0d2')
hashlib_events = [audio_premis_checksum_uuid, premis_checksum_uuid, package_manifest_uuid]
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
brian_events = [extract_uuid]
if user == 'Brian Cash':
brian_events += audio_premis_checksum_uuid,audio_framemd5_uuid,premis_checksum_uuid,package_manifest_uuid
brian_events = [capture_received_uuid, scanning_uuid]
elif user == 'Gavin Martin':
brian_events += audio_premis_checksum_uuid,audio_framemd5_uuid,premis_checksum_uuid,package_manifest_uuid
gavin_events = [capture_received_uuid, scanning_uuid]
gavinAgent = make_agent(premis,gavin_events, '9cab0b9c-4787-4482-8927-a045178c8e39')
if user == 'Gavin Martin':
script_user_Agent = gavinAgent
brianAgent = make_agent(premis,brian_events, '0b96a20d-49f5-46e9-950d-4e11242a487e')
if user == 'Brian Cash':
script_user_Agent = brianAgent
if aeolight_workstation == 'telecine':
macMiniTelecineMachineAgent_events = [extract_uuid, audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid,package_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [extract_uuid, audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid, package_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
transcoderMachine = make_agent(premis,[capture_received_uuid], '946e5d40-a07f-47d1-9637-def5cb7854ba')
transcoderMachineOS = make_agent(premis,[capture_received_uuid], '192f61b1-8130-4236-a827-a194a20557fe')
aeolight_computer = macMiniTelecineMachineAgent
aeolight_OS = macMiniTelecineOSAgent
elif aeolight_workstation == 'ca_machine':
macMiniTelecineMachineAgent_events = [audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid,package_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid, package_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
transcoderMachine = make_agent(premis,[capture_received_uuid, extract_uuid], '946e5d40-a07f-47d1-9637-def5cb7854ba')
transcoderMachineOS = make_agent(premis,[capture_received_uuid, extract_uuid], '192f61b1-8130-4236-a827-a194a20557fe')
aeolight_computer = transcoderMachine
aeolight_OS = transcoderMachineOS
ffmpegAgent_events = [audio_framemd5_uuid]
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
scannerAgent = make_agent(premis,[scanning_uuid], '1f4c1369-e9d1-425b-a810-6db1150955ba')
scannerPCAgent = make_agent(premis,[scanning_uuid], 'ca731b64-638f-4dc3-9d27-0fc14387e38c')
scannerLinuxAgent = make_agent(premis,[scanning_uuid], 'b22baa5c-8160-427d-9e2f-b62a7263439d')
make_event(premis, 'creation', 'Film scanned to 12-bit RAW Bayer format and transcoded internally by ca731b64-638f-4dc3-9d27-0fc14387e38c to 16-bit RGB linear TIFF', [scannerAgent, script_user_Agent, scannerPCAgent, scannerLinuxAgent], scanning_uuid,xml_info[4], 'outcome')
make_event(premis, 'creation', 'TIFF image sequence is received via ethernet from ca731b64-638f-4dc3-9d27-0fc14387e38c and written to Disk', [transcoderMachine,transcoderMachineOS, script_user_Agent], capture_received_uuid,image_uuids,'outcome')
make_event(premis, 'creation', 'PCM WAV file extracted from overscanned image area of source TIFF files', [aeolightAgent, brianAgent, aeolight_computer, aeolight_OS ], extract_uuid,[audio_file_uuid], 'outcome')
make_event(premis, 'message digest calculation', 'Whole file checksum of audio created for PREMIS XML', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], audio_premis_checksum_uuid,[audio_file_uuid], 'source')
make_event(premis, 'message digest calculation', 'Frame level checksums of audio', [ffmpegAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], audio_framemd5_uuid,[audio_file_uuid], 'source' )
make_event(premis, 'message digest calculation', 'Whole file checksums of image created for PREMIS XML', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], premis_checksum_uuid,[representation_uuid], 'source')
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], package_manifest_uuid,[representation_uuid], 'source' )
write_premis(doc, premisxml)
def premis_description(root_dir, process_counter, total_process, aeo_raw_extract_wav_dir, user, aeolight_workstation, audio_date_modified, intellectual_entity_uuid):
source_directory = root_dir + '/objects/image'
image_dir_list = os.listdir(source_directory)
for image_files in image_dir_list:
if not image_files[0] == '.':
if image_files.endswith('.tiff'):
first_image = source_directory + '/' + image_files
image_date_modified = get_date_modified(first_image)
continue
print 'Process %d of %d - Generating PREMIS XML file' % (process_counter,total_process)
process_counter += 1
representation_uuid = str(uuid.uuid4())
premisxml, premis_namespace, doc, premis = setup_xml(source_directory)
split_list = os.path.basename(root_dir).split('_')
if 'ifard2016' in split_list[2]:
split_list[2] = split_list[2].replace('ifard2016', 'IFA-(RD)2016-')
elif 'ifard2017' in split_list[2]:
split_list[2] = split_list[2].replace('ifard2017', 'IFA-(RD)2017-')
audio_items = {"workflow":"raw audio","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'******'}
image_items = {"workflow":"scanning","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
linking_representation_uuids = []
xml_info = make_premis(aeo_raw_extract_wav_dir, audio_items, premis, premis_namespace, premisxml, representation_uuid, 'nosequence')
linking_representation_uuids.append(xml_info[2])
xml_info = make_premis(source_directory, image_items, premis, premis_namespace,premisxml, representation_uuid, 'sequence')
image_uuids = xml_info[4]
linking_representation_uuids.append(image_uuids)
linking_representation_uuids.append(image_items['sourceAccession'])
audio_file_uuid = linking_representation_uuids[0]
create_intellectual_entity(premisxml, premis_namespace, doc, premis, audio_items, intellectual_entity_uuid)
create_representation(premisxml, premis_namespace, doc, premis, audio_items,linking_representation_uuids, representation_uuid, 'sequence', intellectual_entity_uuid)
doc = xml_info[0]
premisxml = xml_info[1]
premis = doc.getroot()
extract_uuid = str(uuid.uuid4())
capture_received_uuid = str(uuid.uuid4())
audio_premis_checksum_uuid = str(uuid.uuid4())
audio_framemd5_uuid = str(uuid.uuid4())
scanning_uuid = str(uuid.uuid4())
premis_checksum_uuid = str(uuid.uuid4())
package_manifest_uuid = str(uuid.uuid4())
aeolight_events = [extract_uuid]
aeolightAgent = make_agent(premis,aeolight_events, 'bc3de900-3903-4764-ab91-2ce89977d0d2')
hashlib_events = [audio_premis_checksum_uuid, premis_checksum_uuid, package_manifest_uuid]
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
brian_events = [extract_uuid]
if user == 'Brian Cash':
brian_events += audio_premis_checksum_uuid,audio_framemd5_uuid,premis_checksum_uuid,package_manifest_uuid
brian_events = [capture_received_uuid, scanning_uuid]
elif user == 'Gavin Martin':
brian_events += audio_premis_checksum_uuid,audio_framemd5_uuid,premis_checksum_uuid,package_manifest_uuid
gavin_events = [capture_received_uuid, scanning_uuid]
gavinAgent = make_agent(premis,gavin_events, '9cab0b9c-4787-4482-8927-a045178c8e39')
if user == 'Gavin Martin':
script_user_Agent = gavinAgent
brianAgent = make_agent(premis,brian_events, '0b96a20d-49f5-46e9-950d-4e11242a487e')
if user == 'Brian Cash':
script_user_Agent = brianAgent
if aeolight_workstation == 'telecine':
macMiniTelecineMachineAgent_events = [extract_uuid, audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid,package_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [extract_uuid, audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid, package_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
transcoderMachine = make_agent(premis,[capture_received_uuid], '946e5d40-a07f-47d1-9637-def5cb7854ba')
transcoderMachineOS = make_agent(premis,[capture_received_uuid], '192f61b1-8130-4236-a827-a194a20557fe')
aeolight_computer = macMiniTelecineMachineAgent
aeolight_OS = macMiniTelecineOSAgent
elif aeolight_workstation == 'ca_machine':
macMiniTelecineMachineAgent_events = [audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid,package_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [audio_premis_checksum_uuid, premis_checksum_uuid, audio_framemd5_uuid, package_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
transcoderMachine = make_agent(premis,[capture_received_uuid, extract_uuid], '946e5d40-a07f-47d1-9637-def5cb7854ba')
transcoderMachineOS = make_agent(premis,[capture_received_uuid, extract_uuid], '192f61b1-8130-4236-a827-a194a20557fe')
aeolight_computer = transcoderMachine
aeolight_OS = transcoderMachineOS
ffmpegAgent_events = [audio_framemd5_uuid]
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
scannerAgent = make_agent(premis,[scanning_uuid], '1f4c1369-e9d1-425b-a810-6db1150955ba')
scannerPCAgent = make_agent(premis,[scanning_uuid], 'ca731b64-638f-4dc3-9d27-0fc14387e38c')
scannerLinuxAgent = make_agent(premis,[scanning_uuid], 'b22baa5c-8160-427d-9e2f-b62a7263439d')
make_event(premis, 'creation', 'Film scanned to 12-bit RAW Bayer format and transcoded internally by ca731b64-638f-4dc3-9d27-0fc14387e38c to 16-bit RGB linear TIFF', [scannerAgent, script_user_Agent, scannerPCAgent, scannerLinuxAgent], scanning_uuid,xml_info[4], 'outcome', image_date_modified)
make_event(premis, 'creation', 'TIFF image sequence is received via ethernet from ca731b64-638f-4dc3-9d27-0fc14387e38c and written to Disk', [transcoderMachine,transcoderMachineOS, script_user_Agent], capture_received_uuid,image_uuids,'outcome', image_date_modified)
make_event(premis, 'creation', 'PCM WAV file extracted from overscanned image area of source TIFF files', [aeolightAgent, brianAgent, aeolight_computer, aeolight_OS ], extract_uuid,[audio_file_uuid], 'outcome',audio_date_modified)
make_event(premis, 'message digest calculation', 'Whole file checksum of audio created for PREMIS XML', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], audio_premis_checksum_uuid,[audio_file_uuid], 'source', 'now')
make_event(premis, 'message digest calculation', 'Frame level checksums of audio', [ffmpegAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], audio_framemd5_uuid,[audio_file_uuid], 'source', 'now' )
make_event(premis, 'message digest calculation', 'Whole file checksums of image created for PREMIS XML', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], premis_checksum_uuid,[representation_uuid], 'source', 'now')
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [hashlibAgent, brianAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent], package_manifest_uuid,[representation_uuid], 'source', 'now' )
write_premis(doc, premisxml)
def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n' )
while permission not in ('Y','y','N','n'):
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n')
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission =='y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename, ('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root,dirnames,filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root# + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root,files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename( master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename( mezzanine_parent_dir) + '_manifest.md5'
master_audio = master_parent_dir + '/objects/audio/' + os.listdir(master_parent_dir + '/objects/audio')[0]
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
source_representation_uuid = premis_description(master_object_dir, master_parent_dir + '/objects/audio', user)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir,audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores= ['ffmpeg','-y','-f','image2','-framerate','24', '-start_number', start_number, '-i', root + '/' + dpx_filename ,'-i', audio_file,'-c:v','prores','-profile:v', '3','-c:a','pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov','-f', 'framemd5', '-an', master_metadata_dir + '/image/' + os.path.basename(master_parent_dir) + '.framemd5', '-c:a', 'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir + '/audio/' + os.path.basename(master_parent_dir) + '.framemd5']
print seq2prores
subprocess.call(seq2prores,env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(mezzanine_file)
items = {"workflow":"seq2prores","oe":'n/a', "filmographic":split_list[0], "sourceAccession":split_list[1], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis, premis_namespace,premisxml, representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(xml_info[2]) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_representation(premisxml, premis_namespace, doc, premis, items,linking_representation_uuids, representation_uuid,sequence )
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid ]
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid ]
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid,prores_event_uuid]
operatorAgent = make_agent(premis,operatorEvents ,user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(premis, 'creation', 'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, ffmpegAgent, operatorAgent ],prores_event_uuid,[representation_uuid], 'outcome')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
tracexml = "%s/%s_mediatrace.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
audio_mediatracexml = "%s/%s_mediatrace.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml,'audiomediaxmlinput',master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml,'audiomediatraceinput',master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(mezzanine_mediainfoxml,'mediaxmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(tracexml,'mediatracexmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest, master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest, mezzanine_parent_dir)
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, operatorAgent],final_sip_manifest_uuid,[representation_uuid], 'source')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename, (os.path.basename( master_parent_dir), start, finish))
'''
def premis_description(root_dir, aeo_raw_extract_wav_dir, user):
source_directory = root_dir
representation_uuid = str(uuid.uuid4())
premisxml, premis_namespace, doc, premis = setup_xml(source_directory)
split_list = os.path.basename(os.path.dirname(os.path.dirname(root_dir))).split('_')
if 'ifard2016' in split_list[2]:
split_list[2] = split_list[2].replace('ifard2016', 'IFA-(RD)2016-')
audio_items = {"workflow":"treated audio","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'******'}
image_items = {"workflow":"grade","oe":split_list[0], "filmographic":split_list[1], "sourceAccession":split_list[2], "interventions":['placeholder'], "prepList":['placeholder'], "user":'******'}
linking_representation_uuids = []
xml_info = make_premis(aeo_raw_extract_wav_dir, audio_items, premis, premis_namespace, premisxml, representation_uuid, 'nosequence')
linking_representation_uuids.append(xml_info[2])
xml_info = make_premis(source_directory, image_items, premis, premis_namespace,premisxml, representation_uuid, 'sequence')
print xml_info
linking_representation_uuids.append(xml_info[4])
linking_representation_uuids.append(image_items['sourceAccession'])
create_representation(premisxml, premis_namespace, doc, premis, audio_items,linking_representation_uuids, representation_uuid, 'sequence' )
doc = xml_info[0]
premisxml = xml_info[1]
premis = doc.getroot()
'''
events:
audio - audio cleaning in rx5
export from protools
image:
crop in avid
grade in baselight
export from avid
framemd5 audio
framemd5 image
whole md5
'''
audio_rx5_uuid = str(uuid.uuid4())
audio_protools_uuid = str(uuid.uuid4())
image_avid_crop_uuid = str(uuid.uuid4())
image_baselight_grade_uuid = str(uuid.uuid4())
package_manifest_uuid = str(uuid.uuid4())
image_framemd5_uuid = str(uuid.uuid4())
ffmpegAgent_events = [image_framemd5_uuid ]
hashlib_events = [package_manifest_uuid]
avid_events = [image_avid_crop_uuid,image_baselight_grade_uuid]
protools_events = [audio_protools_uuid]
baselight_events = [image_baselight_grade_uuid]
rx5_events = [audio_rx5_uuid]
macMiniTelecineMachineAgent_events = [audio_rx5_uuid, package_manifest_uuid, image_framemd5_uuid, audio_protools_uuid]
macMiniTelecineMachineOSAgent_events = [audio_rx5_uuid, package_manifest_uuid, image_framemd5_uuid, audio_protools_uuid]
macProTelecineMachineOSAgent_events = [image_avid_crop_uuid, image_baselight_grade_uuid]
macProTelecineMachineAgent_events = [image_avid_crop_uuid, image_baselight_grade_uuid]
gavin_events = [image_avid_crop_uuid, image_baselight_grade_uuid]
brian_events = [audio_rx5_uuid, audio_protools_uuid]
if user == 'Gavin Martin':
gavin_events += package_manifest_uuid, image_framemd5_uuid
elif user == 'Brian Cash':
brian_events += package_manifest_uuid, image_framemd5_uuid
else:
script_user_events = [package_manifest_uuid, image_framemd5_uuid,]
script_user_Agent = make_agent(premis,script_user_events, user )
gavinAgent = make_agent(premis,gavin_events, '9cab0b9c-4787-4482-8927-a045178c8e39')
if user == 'Gavin Martin':
script_user_Agent = gavinAgent
brianAgent = make_agent(premis,brian_events, '0b96a20d-49f5-46e9-950d-4e11242a487e')
if user == 'Brian Cash':
script_user_Agent = brianAgent
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macProTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '838a1a1b-7ddd-4846-ae8e-3b5ecb4aae55')
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '68f56ede-a1cf-48aa-b1d8-dc9850d5bfcc')
macProTelecineOSAgent = make_agent(premis,macProTelecineMachineOSAgent_events, '52adf876-bf30-431c-b0c6-80cc4fd9406c')
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
avidAgent = make_agent(premis,avid_events, '11e157a3-1aa7-4195-b816-009a3d47148c')
protoolsAgent = make_agent(premis,protools_events, '55003bbd-49a4-4c7b-8da2-0d5b9bf10168')
baselightAgent = make_agent(premis,baselight_events, '8c02d962-5ac5-4e51-a30c-002553134320')
rx5Agent = make_agent(premis,rx5_events, 'e5872957-8ee8-4c20-bd8e-d76e1de01b34')
make_event(premis, 'creation', 'Audio cleanup', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, rx5Agent , brianAgent ],audio_rx5_uuid,[linking_representation_uuids[0]], 'outcome')
make_event(premis, 'creation', 'Audio trimming and export', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, protoolsAgent, brianAgent ],audio_protools_uuid ,[linking_representation_uuids[0]], 'outcome')
make_event(premis, 'creation', 'Import to Avid and remove overscan', [macProTelecineMachineAgent ,macProTelecineOSAgent, avidAgent, gavinAgent ],image_avid_crop_uuid,xml_info[4], 'outcome')
make_event(premis, 'creation', 'Colour Correction', [macProTelecineMachineAgent ,macProTelecineOSAgent, baselightAgent , gavinAgent ],image_baselight_grade_uuid ,xml_info[4], 'outcome')
make_event(premis, 'message digest calculation', 'Frame level checksums of image', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, ffmpegAgent, script_user_Agent ],image_framemd5_uuid,xml_info[4], 'source')
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [hashlibAgent,macMiniTelecineMachineAgent, macMiniTelecineOSAgent,script_user_Agent], package_manifest_uuid,[representation_uuid], 'source' )
write_premis(doc, premisxml)
return representation_uuid