def short_test(images, args):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
maybe all that needs to happen is that 24 frames are copied to a temp
location, then the functions run, and use ififuncs to check the sums
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
for image in images[:24]:
full_path = os.path.join(args.i, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
ififuncs.diff_textfiles(converted_manifest, restored_manifest)
def create_manifest(source):
'''
Generates a master log and creates checksum manifests for all subdirectories.
'''
master_log = os.path.expanduser('~/Desktop/batchfixity_errors.log')
os.chdir(source)
for dirname in os.walk('.').next()[1]:
full_path = os.path.join(source, dirname)
manifest_textfile = '%s/%s_manifest.md5' % (full_path, dirname)
if not os.path.isfile(manifest_textfile):
log_name = '%s/%s_fixity.log' % (
os.path.dirname(full_path), dirname
)
generate_log(log_name, 'batchfixity started')
generate_log(log_name, '%s created' % manifest_textfile)
try:
hashlib_manifest(full_path, manifest_textfile, full_path)
generate_log(log_name, 'manifest creation complete')
shutil.move(log_name, full_path)
except IOError:
with open(master_log, 'ab') as log:
log.write(
'%s has failed probably because of special characters like a fada\n' % full_path
)
generate_log(
log_name, 'manifest has failed probably because of special characters like a fada'
)
def create_manifest(source):
'''
Generates a master log and creates checksum manifests for all subdirectories.
'''
master_log = os.path.expanduser('~/Desktop/batchfixity_errors.log')
os.chdir(source)
for dirname in os.walk('.').next()[1]:
full_path = os.path.join(source, dirname)
manifest_textfile = '%s/%s_manifest.md5' % (full_path, dirname)
if not os.path.isfile(manifest_textfile):
log_name = '%s/%s_fixity.log' % (os.path.dirname(full_path),
dirname)
generate_log(log_name, 'batchfixity started')
generate_log(log_name, '%s created' % manifest_textfile)
try:
hashlib_manifest(full_path, manifest_textfile, full_path)
generate_log(log_name, 'manifest creation complete')
shutil.move(log_name, full_path)
except IOError:
with open(master_log, 'ab') as log:
log.write(
'%s has failed probably because of special characters like a fada\n'
% full_path)
generate_log(
log_name,
'manifest has failed probably because of special characters like a fada'
)
def make_sip(video_files):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_mediainfo.xml" % (
metadata_dir, os.path.basename(filename)
)
inputtracexml = "%s/%s_mediatrace.xml" % (
metadata_dir, os.path.basename(filename)
)
fmd5 = "%s/%s.framemd5" % (
metadata_dir, os.path.basename(filename)
)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
fmd5_logfile = log_dir + '/%s_framemd5.log' % filename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', filename,
'-report',
'-f', 'framemd5', '-an',
fmd5
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log,
'makeffv1.py Framemd5 generation of output file completed'
)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
if os.path.isfile(filename):
shutil.move(filename, data_dir)
generate_log(log, 'dvsip.py DV file moved to %s' % data_dir)
generate_log(log, 'dvsip.py MD5 manifest started')
hashlib_manifest(filenoext, manifest, source_parent_dir)
def make_sip(video_files):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = '%s/metadata' % filenoext
log_dir = '%s/logs' % filenoext
data_dir = '%s/objects' % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = '%s/%s_mediainfo.xml' % (metadata_dir,
os.path.basename(filename))
inputtracexml = '%s/%s_mediatrace.xml' % (metadata_dir,
os.path.basename(filename))
fmd5 = '%s/%s.framemd5' % (metadata_dir, os.path.basename(filename))
log = '%s/%s_log.log' % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
fmd5_logfile = log_dir + '/%s_framemd5.log' % filename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i',
filename,
'-report',
'-f',
'framemd5',
'-an',
fmd5
]
print(fmd5_command)
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file completed')
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print('Generating mediainfo xml of input file and saving it in %s'
) % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print('Generating mediatrace xml of input file and saving it in %s'
) % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
if os.path.isfile(filename):
shutil.move(filename, data_dir)
generate_log(log, 'dvsip.py DV file moved to %s' % data_dir)
generate_log(log, 'dvsip.py MD5 manifest started')
hashlib_manifest(filenoext, manifest, source_parent_dir)
def process_audio(input, args):
total_process = 6
desktop_dir = os.path.expanduser("~/Desktop/%s") % os.path.basename(input)
parent_dir = os.path.dirname(input)
root_dir = os.path.dirname(os.path.dirname(parent_dir))
os.chdir(root_dir)
metadata_dir = root_dir + '/metadata/audio'
aeo_light_premislog = input + '.xml'
if os.path.isfile(aeo_light_premislog):
shutil.move(aeo_light_premislog, metadata_dir)
logs_dir = 'logs/audio'
aeo_raw_extract_wav_dir = root_dir + '/objects/audio'
framemd5 = metadata_dir + '/' + os.path.basename(input) +'.framemd5'
if os.path.isfile(framemd5):
return 'x', 'x', 'x', 'x'
logfile = logs_dir + '/%s_framemd5.log' % os.path.basename(input)
process_counter = 1
print 'Process %d of %d - Logfile of framemd5 ffmpeg process located in %s/%s' % (process_counter,total_process, root_dir, logfile)
process_counter += 1
normpath = os.path.normpath(root_dir)
relative_path = normpath.split(os.sep)[-1]
manifest = root_dir + '/raw_sip_manifest.md5'
filenoext = os.path.splitext(input)[0]
inputxml = inputxml = "%s/%s_mediainfo.xml" % (metadata_dir,os.path.basename(input))
make_mediainfo(inputxml,'mediaxmlinput',input)
env_dict = set_environment(logfile)
ffmpegcmd = ['ffmpeg', # Create decoded md5 checksums for every frame of the ffv1 output
'-i',input,
'-report',]
if not args.v:
ffmpegcmd += ['-v', '0']
ffmpegcmd += ['-f','framemd5',
framemd5 ]
print 'Process %d of %d - Generating framemd5 values for source WAV' % (process_counter,total_process)
process_counter += 1
subprocess.call(ffmpegcmd,env=env_dict)
print 'Process %d of %d - Creating a workhorse copy of source WAV on Desktop' % (process_counter,total_process)
process_counter += 1
shutil.copy(input, desktop_dir)
print 'Process %d of %d - Checking if any unwanted files should be removed, eg .DS_Stores or desktop.ini/thumbs.db' % (process_counter,total_process)
process_counter += 1
remove_bad_files(root_dir)
os.chdir(parent_dir)
print 'Process %d of %d - Generating manifest' % (process_counter,total_process)
process_counter += 1
hashlib_manifest(root_dir, manifest, root_dir)
return root_dir, process_counter, total_process, aeo_raw_extract_wav_dir
def reversibility_verification(objects, source_manifest, reversibility_dir):
'''
Restore the MKV back to DPX, create a checksum, and compare to source DPX
checksums.
Return a value of lossy or lossless.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(reversibility_dir, temp_uuid)
os.makedirs(temp_dir)
for ffv1_mkv in objects:
subprocess.call(['rawcooked', ffv1_mkv, '-o', temp_dir])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, source_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Unable to delete temp directory, sorry!')
return judgement
def short_test(images):
'''
Perform a test on the first 24 frames that will encode via Rawcooked,
then decode back to the original form,
and the whole file checksums of the original 24 frames
and the restored 24 frames are compared.
'''
temp_uuid = ififuncs.create_uuid()
temp_dir = os.path.join(tempfile.gettempdir(), temp_uuid)
os.makedirs(temp_dir)
# Only analyse the first 24 frames.
for image in sorted(os.listdir(images))[:24]:
full_path = os.path.join(images, image)
shutil.copy(full_path, temp_dir)
mkv_uuid = ififuncs.create_uuid()
mkv_file = os.path.join(tempfile.gettempdir(), mkv_uuid + '.mkv')
subprocess.call(['rawcooked', temp_dir, '-c:a', 'copy', '-o', mkv_file])
converted_manifest = os.path.join(temp_dir, '123.md5')
ififuncs.hashlib_manifest(temp_dir, converted_manifest, temp_dir)
subprocess.call(['rawcooked', mkv_file])
rawcooked_dir = mkv_file + '.RAWcooked'
restored_dir = os.path.join(rawcooked_dir, temp_uuid)
restored_manifest = os.path.join(restored_dir, '456.md5')
ififuncs.hashlib_manifest(restored_dir, restored_manifest, restored_dir)
judgement = ififuncs.diff_textfiles(converted_manifest, restored_manifest)
print((' - Deleting temp directory %s' % temp_dir))
try:
shutil.rmtree(temp_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp reversibility directory %s' % rawcooked_dir))
try:
shutil.rmtree(rawcooked_dir)
except WindowsError:
print('Sorry, we do not have permission to delete these files')
print((' - Deleting temp FFV1/MKV %s' % mkv_file))
os.remove(mkv_file)
return judgement
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
clairmeta_version = clairmeta.__version__
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print(args)
user = ififuncs.determine_user(args)
object_entry = get_object_entry(args)
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
uuid, uuid_event = determine_uuid(args, sip_path)
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
if args.d:
content_title = create_content_title_text(sip_path, args)
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
if args.accession:
accession_number = ififuncs.get_accession_number()
reference_number = ififuncs.get_reference_number()
parent = ififuncs.ask_question(
'What is the parent record? eg MV 1234. Enter n/a if this is a born digital acquisition with no parent.'
)
donor = ififuncs.ask_question(
'Who is the source of acquisition, as appears on the donor agreement? This will not affect Reproductions.'
)
depositor_reference = ififuncs.ask_question(
'What is the donor/depositor number? This will not affect Reproductions.'
)
acquisition_type = ififuncs.get_acquisition_type('')
donation_date = ififuncs.ask_question(
'When was the donation date in DD/MM/YYYY format? Eg. 31/12/1999 - Unfortunately this is NOT using ISO 8601.'
)
if args.zip:
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(
inputs[0], args.o, uuid, new_log_textfile)
if args.manifest:
shutil.copy(
args.manifest,
args.manifest.replace('_manifest.md5', '_manifest-md5.txt'))
source_manifest = args.manifest.replace('_manifest.md5',
'_manifest-md5.txt')
else:
source_manifest = os.path.join(
args.o,
os.path.basename(args.i[0]) + '_manifest-md5.txt')
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.hashlib_manifest(args.i[0], source_manifest,
os.path.dirname(args.i[0]))
ififuncs.generate_log(
new_log_textfile,
'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files within ZIP'
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=started, eventType=packing, agentName=makezip.py, eventDetail=Source object to be packed=%s'
% inputs[0])
makezip_judgement, zip_file = makezip.main([
'-i', inputs[0], '-o',
os.path.join(sip_path, 'objects'), '-basename', uuid + '.zip'
])
ififuncs.generate_log(
new_log_textfile,
'EVENT = packing, status=finished, eventType=packing, agentName=makezip.py, eventDetail=Source object packed into=%s'
% zip_file)
if makezip_judgement is None:
judgement = 'lossless'
else:
judgement = makezip_judgement
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
ififuncs.generate_log(
new_log_textfile,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=makezip.py, eventDetail=embedded crc32 checksum validation, eventOutcome=%s'
% judgement)
else:
log_names = move_files(inputs, sip_path, args, user)
ififuncs.get_technical_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
if args.zip:
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=started, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
ififuncs.manifest_update(new_manifest_textfile, zip_file)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Message Digest Calculation, status=finished, eventType=message digest calculation, eventDetail=%s module=hashlib'
% zip_file)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.zip:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', [inputxml, inputtracexml, dfxml, source_manifest], '-user',
user, '-new_folder', supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
if 'log_names' in locals():
log_report(log_names)
finish = datetime.datetime.now()
print('\n- %s ran this script at %s and it finished at %s' %
(user, start, finish))
if args.d:
process_dcp(sip_path, content_title, args, new_manifest_textfile,
new_log_textfile, metadata_dir, clairmeta_version)
if args.accession:
register = accession.make_register()
filmographic_dict = ififuncs.extract_metadata(args.csv)[0]
for filmographic_record in filmographic_dict:
if filmographic_record['Reference Number'].lower(
) == reference_number.lower():
if filmographic_record['Title'] == '':
title = filmographic_record[
'TitleSeries'] + '; ' + filmographic_record['EpisodeNo']
else:
title = filmographic_record['Title']
oe_register = make_oe_register()
ififuncs.append_csv(
oe_register,
(object_entry.upper()[:2] + '-' + object_entry[2:], donation_date,
'1', '', title, donor, acquisition_type[1], accession_number,
'Representation of %s|Reproduction of %s' %
(reference_number, parent), ''))
accession_cmd = [
os.path.dirname(sip_path), '-user', user, '-f', '-number',
accession_number, '-reference', reference_number, '-register',
register, '-csv', args.csv, '-pbcore'
]
if not parent.lower() == 'n/a':
accession_cmd.extend(['-parent', parent])
accession_cmd.extend(['-donor', donor])
accession_cmd.extend(['-depositor_reference', depositor_reference])
accession_cmd.extend(['-acquisition_type', acquisition_type[2]])
accession_cmd.extend(['-donation_date', donation_date])
print(accession_cmd)
accession.main(accession_cmd)
return new_log_textfile, new_manifest_textfile
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) != 4:
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid
)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(
new_log_textfile,
'EVENT = sipcreator.py started'
)
ififuncs.generate_log(
new_log_textfile,
'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')
)
ififuncs.generate_log(
new_log_textfile,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = agentName=%s' % user
)
ififuncs.generate_log(
new_log_textfile,
uuid_event
)
if args.u is False:
sys.exit()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s'
% object_entry
)
metadata_dir = os.path.join(sip_path, 'metadata')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(
metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir
)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(
logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir))
)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish)
if args.d:
content_title = create_content_title_text(args, sip_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"),
os.path.join('objects', content_title).replace("\\", "/")
)
return new_log_textfile, new_manifest_textfile
def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
while permission not in ('Y', 'y', 'N', 'n'):
permission = raw_input(
'\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n'
)
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission == 'y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime(
"_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename,
('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root, dirnames, filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root # + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root, files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(
os.path.dirname(os.path.dirname(master_parent_dir)) +
'/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(
os.path.dirname(
master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(
os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename(
master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_manifest.md5'
audio_dir_list = os.listdir(master_parent_dir + '/objects/audio')
for audio_files in audio_dir_list:
if not audio_files[0] == '.':
if audio_files.endswith('.wav'):
master_audio = master_parent_dir + '/objects/audio/' + audio_files
audio_date_modified = get_date_modified(master_audio)
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(
mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
image_date_modified = info[3]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
intellectual_entity_uuid = str(uuid.uuid4())
source_representation_uuid = premis_description(
master_object_dir, master_parent_dir + '/objects/audio', user,
image_date_modified, audio_date_modified,
intellectual_entity_uuid)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir, audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(
mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores = [
'ffmpeg', '-y', '-f', 'image2', '-framerate', '24',
'-start_number', start_number, '-i', root + '/' + dpx_filename,
'-i', audio_file, '-c:v', 'prores', '-profile:v', '3', '-c:a',
'pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov',
'-f', 'framemd5', '-an', master_metadata_dir + '/image/' +
os.path.basename(master_parent_dir) + '.framemd5', '-c:a',
'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir +
'/audio/' + os.path.basename(master_parent_dir) + '.framemd5'
]
print seq2prores
subprocess.call(seq2prores, env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(
mezzanine_file)
items = {
"workflow": "seq2prores",
"oe": 'n/a',
"filmographic": split_list[0],
"sourceAccession": split_list[1],
"interventions": ['placeholder'],
"prepList": ['placeholder'],
"user": user
}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis,
premis_namespace, premisxml,
representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(
xml_info[2]
) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_intellectual_entity(premisxml, premis_namespace, doc,
premis, items, intellectual_entity_uuid)
create_representation(premisxml, premis_namespace, doc, premis,
items, linking_representation_uuids,
representation_uuid, sequence,
intellectual_entity_uuid)
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineMachineAgent = make_agent(
premis, macMiniTelecineMachineAgent_events,
'230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [
prores_event_uuid, final_sip_manifest_uuid
]
macMiniTelecineOSAgent = make_agent(
premis, macMiniTelecineMachineOSAgent_events,
'9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid]
hashlibAgent = make_agent(premis, hashlib_events,
'9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid]
ffmpegAgent = make_agent(premis, ffmpegAgent_events,
'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid, prores_event_uuid]
operatorAgent = make_agent(premis, operatorEvents, user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(
premis, 'creation',
'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio',
[
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
ffmpegAgent, operatorAgent
], prores_event_uuid, [representation_uuid], 'outcome', 'now')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
tracexml = "%s/%s_mediatrace.xml" % (
mezzanine_metadata_dir, os.path.basename(mezzanine_parent_dir))
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
audio_mediatracexml = "%s/%s_mediatrace.xml" % (
master_metadata_dir + '/audio', os.path.basename(master_audio))
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml, 'audiomediaxmlinput',
master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml, 'audiomediatraceinput',
master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(
mezzanine_mediainfoxml, 'mediaxmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(
tracexml, 'mediatracexmlinput',
mezzanine_object_dir + '/' +
os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest,
master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest,
mezzanine_parent_dir)
make_event(premis, 'message digest calculation',
'Checksum manifest for whole package created', [
macMiniTelecineMachineAgent, macMiniTelecineOSAgent,
operatorAgent
], final_sip_manifest_uuid, [representation_uuid],
'source', 'now')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename,
(os.path.basename(master_parent_dir), start, finish))
'''
def main(args_):
'''
Overly long main function that makes a sidecar manifest.
This needs to get broken up into smaller functions.
'''
parser = argparse.ArgumentParser(description='Generate manifest with'
' checksums for a directory'
' Written by Kieran O\'Leary.')
parser.add_argument(
'source',
help='Input directory'
)
parser.add_argument(
'-s', '-sidecar',
action='store_true',
help='Generates Sidecar'
)
parser.add_argument(
'-f', '-felix',
action='store_true',
help='Felix Meehan workflow - places manifest inside of source directory'
)
parser.add_argument(
'-sha512',
action='store_true',
help='Generates sha512 checksums instead of md5'
)
args = parser.parse_args(args_)
source = args.source
source_parent_dir = os.path.dirname(source)
normpath = os.path.normpath(source)
relative_path = normpath.split(os.sep)[-1]
log_name_source_ = os.path.basename(
args.source
) + time.strftime("_%Y_%m_%dT%H_%M_%S")
if args.s:
if args.sha512:
manifest = source_parent_dir + '/%s_manifest-sha512.txt' % relative_path
else:
manifest = source_parent_dir + '/%s_manifest.md5' % relative_path
log_name_source = source_parent_dir + '/%s.log' % log_name_source_
elif args.f:
if args.sha512:
manifest = source_parent_dir + '/%s_manifest-sha512.txt' % relative_path
else:
manifest = source + '/%s_manifest.md5' % relative_path
log_name_source = source_parent_dir + '/%s.log' % log_name_source_
else:
if args.sha512:
manifest_ = manifest_ = '/%s_manifest-sha512.txt' % relative_path
else:
manifest_ = '/%s_manifest.md5' % relative_path
desktop_manifest_dir = make_desktop_manifest_dir()
manifest = "%s/%s" % (desktop_manifest_dir, manifest_)
desktop_logs_dir = make_desktop_logs_dir()
log_name_source = "%s/%s.log" % (desktop_logs_dir, log_name_source_)
if args.sha512:
module = 'hashlib.sha512'
else:
module = 'hashlib.md5'
generate_log(log_name_source, 'manifest.py started.')
if sys.platform == "win32":
generate_log(
log_name_source,
'EVENT = Generating manifest: status=started, eventType=message digest calculation, module=%s, agent=Windows' % module
)
if sys.platform == "darwin":
generate_log(
log_name_source,
'EVENT = Generating manifest: status=started, eventType=message digest calculation, module=%s, agent=OSX' % module
)
elif sys.platform == "linux2":
generate_log(
log_name_source,
'EVENT = Generating manifest: status=started, eventType=message digest calculation, module=%s, agent=Linux' % module
)
ififuncs.generate_log(
log_name_source,
'eventDetail=manifest.py %s' % ififuncs.get_script_version('manifest.py'))
generate_log(log_name_source, 'Source: %s' % source)
if os.path.isfile(source):
print('\nFile checksum is not currently supported, only directories.\n')
generate_log(log_name_source, 'Error: Attempted to generate manifest for file. Only Directories/Folders are currently supported')
generate_log(log_name_source, 'manifest.py exit')
sys.exit()
elif not os.path.isdir(source):
print((' %s is either not a directory or it does not exist' % source))
generate_log(log_name_source, ' %s is either not a directory or it does not exist' % source)
generate_log(log_name_source, 'manifest.py exit')
sys.exit()
remove_bad_files(source, log_name_source)
source_count = 0
for _, _, filenames in os.walk(source):
# There has to be a better way to count the files..
for _ in filenames:
source_count += 1 #works in windows at least
if os.path.isfile(manifest):
count_in_manifest = manifest_file_count(manifest)
if source_count != count_in_manifest:
print('This manifest may be outdated as the number of files in your directory does not match the number of files in the manifest')
generate_log(log_name_source, 'EVENT = Existing source manifest check - Failure - The number of files in the source directory is not equal to the number of files in the source manifest ')
sys.exit()
if not os.path.isfile(manifest):
try:
print('Generating source manifest')
generate_log(log_name_source, 'EVENT = Generating source manifest')
if args.f:
if args.sha512:
ififuncs.sha512_manifest(source, manifest, source)
else:
hashlib_manifest(source, manifest, source)
shutil.move(log_name_source, source)
else:
if args.sha512:
ififuncs.sha512_manifest(source, manifest, source_parent_dir)
else:
hashlib_manifest(source, manifest, source_parent_dir)
except OSError:
print('You do not have access to this directory. Perhaps it is read only, or the wrong file system\n')
sys.exit()
else:
generate_log(log_name_source, 'EVENT = Existing source manifest check - Source manifest already exists. Script will exit. ')
print(('Manifest created in %s' % manifest))
generate_log(log_name_source, 'Manifest created in %s' % manifest)
return log_name_source
def package(objects, object_entry, uuid, source_abspath, args, log_name_source, normalisation_tool, user, rawcooked_logfiles, multi_reeler, current_dir):
'''
Package the MKV using sipcreator.py
'''
sip_dir = os.path.join(
args.o, os.path.join(object_entry, uuid)
)
inputxml, inputtracexml, dfxml = ififuncs.generate_mediainfo_xmls(source_abspath, args.o, uuid, log_name_source)
source_manifest = os.path.join(
args.o,
os.path.basename(args.i) + '_manifest-md5.txt'
)
ififuncs.generate_log(
log_name_source,
'EVENT = message digest calculation, status=started, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files'
)
if multi_reeler:
ififuncs.hashlib_manifest(args.i, source_manifest, args.i)
else:
ififuncs.hashlib_manifest(args.i, source_manifest, os.path.dirname(args.i))
ififuncs.generate_log(
log_name_source,
'EVENT = message digest calculation, status=finished, eventType=messageDigestCalculation, agentName=hashlib, eventDetail=MD5 checksum of source files'
)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibility of %s back to its original form, followed by checksum verification using %s ' % (normalisation_tool, objects, source_manifest)
)
if args.reversibility_dir:
reversibility_dir = args.reversibility_dir
else:
reversibility_dir = args.o
judgement = reversibility_verification(objects, source_manifest, reversibility_dir)
ififuncs.generate_log(
log_name_source,
'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=%s, eventDetail=Full reversibilty of %s back to its original form, followed by checksum verification using %s , eventOutcome=%s' % (normalisation_tool, objects, source_manifest, judgement)
)
supplement_cmd = ['-supplement', inputxml, inputtracexml, dfxml, source_manifest]
if args.supplement:
supplement_cmd.extend(args.supplement)
sipcreator_cmd = [
'-i',
]
for i in objects:
sipcreator_cmd.append(i)
sipcreator_cmd += [
'-u',
uuid,
'-quiet',
'-move',
'-user',
user,
'-oe',
object_entry,
'-o', args.o
]
sipcreator_cmd.extend(supplement_cmd)
sipcreator_log, sipcreator_manifest = sipcreator.main(sipcreator_cmd)
logs_dir = os.path.join(sip_dir, 'logs')
for files in os.listdir(logs_dir):
if files.endswith('.md5'):
deletefiles.main(['-i', os.path.join(logs_dir, files), '-uuid_path', sip_dir, '-user', user])
for rawcooked_logfile in rawcooked_logfiles:
rawcooked_logfile = rawcooked_logfile.replace('\'', '')
shutil.move(rawcooked_logfile, logs_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(logs_dir, os.path.basename(rawcooked_logfile))
)
metadata_dir = os.path.join(sip_dir, 'metadata')
os.chdir(current_dir)
shutil.copy(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'film_scan_aip_documentation.txt'), metadata_dir)
ififuncs.manifest_update(
sipcreator_manifest,
os.path.join(metadata_dir, 'film_scan_aip_documentation.txt')
)
os.remove(dfxml)
os.remove(inputtracexml)
os.remove(inputxml)
return judgement, sipcreator_log, sipcreator_manifest
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (
metadata_dir, os.path.basename(filename)
)
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename)
)
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0]
)
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
fmd5 = "%s/%s_source.framemd5" % (
metadata_dir, os.path.basename(filename)
)
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output(
[
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]
).rstrip()
field_order = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%ScanType%", filename
]
).rstrip()
height = subprocess.check_output(
[
'mediainfo', '--Language=raw',
'--Full', "--Inform=Video;%Height%",
filename
]
).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i', filename,
'-c:v', 'ffv1', # Use FFv1 codec
'-g', '1', # Use intra-frame only aka ALL-I aka GOP=1
'-level', '3', # Use Version 3 of FFv1
'-c:a', 'copy', # Copy and paste audio bitsream with no transcoding
'-map', '0',
'-dn',
'-report',
'-slicecrc', '1',
'-slices', '16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += [
'-vf',
'setfield=tff, setdar=4/3'
]
ffv1_command += [
output,
'-f', 'framemd5', '-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log, 'makeffv1.py transcode to FFV1 and framemd5 generation completed.'
)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.'
)
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i', output,
'-report',
'-f', 'framemd5', '-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log,
'makeffv1.py Framemd5 generation of output file completed'
)
source_video_size = get_mediainfo(
'source_video_size', "--inform=General;%FileSize%", filename
)
ffv1_video_size = get_mediainfo(
'ffv1_video_size', '--inform=General;%FileSize%', output
)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
shutil.copy(sys.argv[0], log_dir)
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)
):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(
csv_report_filename, (
output,
'LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(
output,
'LOSSLESS - different PAR',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(
csv_report_filename,
(
output, 'NOT LOSSLESS',
source_video_size, ffv1_video_size, compression_ratio
)
)
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: # loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
# Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir, os.path.basename(filename))
inputtracexml = "%s/%s_source_mediatrace.xml" % (metadata_dir, os.path.basename(filename))
output = "%s/%s.mkv" % (data_dir, os.path.basename(filename))
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir, outputfilename)
fmd5 = "%s/%s_source.framemd5" % (metadata_dir, os.path.basename(filename))
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, "Input = %s" % filename)
generate_log(log, "Output = %s" % output)
generate_log(log, "makeffv1.py transcode to FFV1 and framemd5 generation of source started.")
ffv1_logfile = log_dir + "/%s_ffv1_transcode.log" % filename
ffv1_env_dict = set_environment(ffv1_logfile)
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
"ffmpeg",
"-i",
filename,
"-c:v",
"ffv1", # Use FFv1 codec
"-g",
"1", # Use intra-frame only aka ALL-I aka GOP=1
"-level",
"3", # Use Version 3 of FFv1
"-c:a",
"copy", # Copy and paste audio bitsream with no transcoding
"-map",
"0",
"-dn",
"-report",
"-slicecrc",
"1",
"-slices",
"16",
output,
"-f",
"framemd5",
"-an", # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5,
]
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(log, "makeffv1.py transcode to FFV1 and framemd5 generation completed.")
generate_log(log, "makeffv1.py Framemd5 generation of output file started.")
fmd5_logfile = log_dir + "/%s_framemd5.log" % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
fmd5_command = [
"ffmpeg", # Create decoded md5 checksums for every frame of the ffv1 output
"-i",
output,
"-report",
"-f",
"framemd5",
"-an",
fmd5ffv1,
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(log, "makeffv1.py Framemd5 generation of output file completed")
source_video_size = get_mediainfo("source_video_size", "--inform=General;%FileSize%", filename)
ffv1_video_size = get_mediainfo("ffv1_video_size", "--inform=General;%FileSize%", output)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1), read_non_comment_lines(f2)
):
if line1 != line2:
if "sar" in line1:
checksum_mismatches = ["sar"]
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print "LOSSLESS"
append_csv(csv_report_filename, (output, "LOSSLESS", source_video_size, ffv1_video_size, compression_ratio))
generate_log(log, "makeffv1.py Transcode was lossless")
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == "sar":
print "Image content is lossless, Pixel Aspect Ratio has been altered"
append_csv(
csv_report_filename,
(output, "LOSSLESS - different PAR", source_video_size, ffv1_video_size, compression_ratio),
)
generate_log(log, "makeffv1.py Image content is lossless, but Pixel Aspect Ratio has been altered")
elif len(checksum_mismatches) > 1:
print "NOT LOSSLESS"
append_csv(
csv_report_filename, (output, "NOT LOSSLESS", source_video_size, ffv1_video_size, compression_ratio)
)
generate_log(log, "makeffv1.py Not Lossless.")
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical. This may be because of a lossy transcode, or a change in metadata, most likely pixel aspect ratio. Please analyse the framemd5 files for source and output."
make_mediainfo(inputxml, "mediaxmlinput", filename)
make_mediainfo(outputxml, "mediaxmloutput", output)
make_mediatrace(inputtracexml, "mediatracexmlinput", filename)
make_mediatrace(outputtracexml, "mediatracexmloutput", output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest_path = os.path.join(source_parent_dir, filenoext)
manifest = "%s/%s_manifest.md5" % (manifest_path, filenoext)
generate_log(log, "makeffv1.py MD5 manifest started")
hashlib_manifest(filenoext, manifest, manifest_path)
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if not args.sc:
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
else:
object_entry = 'not_applicable'
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s'
% uuid)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if args.u is False:
sys.exit()
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
if args.d:
content_title = create_content_title_text(sip_path)
new_dcp_path = os.path.join('objects',
content_title).replace("\\", "/")
absolute_dcp_path = os.path.join(sip_path, new_dcp_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
new_dcp_path)
'''
a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT)
b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT)
'''
dcp = DCP(absolute_dcp_path)
clairmeta_version = clairmeta.__version__
dcp_dict = dcp.parse()
# json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': '))
xml_str = dicttoxml.dicttoxml(dcp_dict,
custom_root='ClairmetaProbe',
ids=False,
attr_type=False)
xml_pretty = prettyprint_xml(xml_str)
status, report = dcp.check()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s'
% (status, report, clairmeta_version))
clairmeta_xml = os.path.join(metadata_dir,
'%s_clairmeta.xml' % content_title)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s'
% (clairmeta_xml, clairmeta_version))
with open(clairmeta_xml, 'w') as fo:
fo.write(xml_pretty)
ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile,
'md5')
ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml)
print status
print report
print '\n', user, 'ran this script at %s and it finished at %s' % (
start, finish)
return new_log_textfile, new_manifest_textfile
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s'
% uuid)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if args.u is False:
sys.exit()
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
if args.d:
content_title = create_content_title_text(args, sip_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
os.path.join('objects', content_title).replace("\\", "/"))
return new_log_textfile, new_manifest_textfile
def main():
desktop_logdir = os.path.expanduser("~/Desktop/") + 'seq_csv_reports'
if not os.path.isdir(desktop_logdir):
os.makedirs(desktop_logdir)
all_files = sys.argv[1:]
permission = ''
if not permission == 'y' or permission == 'Y':
print '\n\n**** All image sequences within these directories will be converted the input for this script.\n'
for i in all_files:
print i
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n' )
while permission not in ('Y','y','N','n'):
permission = raw_input('\n**** All image sequences within these directories will be converted the input for this script \n**** If this looks ok, please press Y, otherwise, type N\n')
if permission == 'n' or permission == 'N':
print 'Exiting at your command- Cheerio for now'
sys.exit()
elif permission =='y' or permission == 'Y':
print 'Ok so!'
csv_report_filename = desktop_logdir + '/seq2prores_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
user = get_user()
create_csv(csv_report_filename, ('Sequence Name', 'Start time', 'Finish Time'))
for source_directory in all_files:
for root,dirnames,filenames in os.walk(source_directory):
#if "tiff_scans" in dirnames:
source_directory = root# + '/tiff_scans'
total_size = 0
remove_bad_files(source_directory)
source_parent_dir = os.path.dirname(source_directory)
normpath = os.path.normpath(source_directory)
relative_path = normpath.split(os.sep)[-1]
split_path = os.path.split(os.path.basename(source_directory))[1]
start = datetime.datetime.now()
info = get_filenames(source_directory, 'dpx_framemd5')
if info == 'none':
continue
for files in filenames:
total_size += os.path.getsize(os.path.join(root,files))
master_parent_dir = os.path.dirname(source_parent_dir)
master_object_dir = master_parent_dir + '/objects/image'
master_metadata_dir = master_parent_dir + '/' + 'metadata'
middle = os.listdir(os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine')[0]
mezzanine_object_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s/objects' % middle
mezzanine_parent_dir = os.path.dirname(os.path.dirname(master_parent_dir)) + '/mezzanine/%s' % middle
mezzanine_metadata_dir = mezzanine_parent_dir + '/metadata'
source_manifest = master_parent_dir + '/' + os.path.basename( master_parent_dir) + '_manifest.md5'
mezzanine_manifest = mezzanine_parent_dir + '/' + os.path.basename( mezzanine_parent_dir) + '_manifest.md5'
master_audio = master_parent_dir + '/objects/audio/' + os.listdir(master_parent_dir + '/objects/audio')[0]
mezzanine_file = mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov'
if os.path.isfile(mezzanine_file):
print 'Mezzanine file already exists so this script has most likely already been run.. skipping.'
continue
image_seq_without_container = info[0]
start_number = info[1]
container = info[2]
start_number_length = len(start_number)
number_regex = "%0" + str(start_number_length) + 'd.'
audio_dir = source_parent_dir + '/audio'
logs_dir = mezzanine_parent_dir + '/logs'
source_representation_uuid = premis_description(master_object_dir, master_parent_dir + '/objects/audio', user)
os.chdir(audio_dir)
audio_file_list = glob('*.wav')
audio_file = os.path.join(audio_dir,audio_file_list[0])
dpx_filename = image_seq_without_container + number_regex + container
logfile = logs_dir + '/%s_prores.log' % os.path.basename(mezzanine_parent_dir)
env_dict = os.environ.copy()
# https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
logfile = "\'" + logfile + "\'"
env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
seq2prores= ['ffmpeg','-y','-f','image2','-framerate','24', '-start_number', start_number, '-i', root + '/' + dpx_filename ,'-i', audio_file,'-c:v','prores','-profile:v', '3','-c:a','pcm_s24le', '-ar', '48000', mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov','-f', 'framemd5', '-an', master_metadata_dir + '/image/' + os.path.basename(master_parent_dir) + '.framemd5', '-c:a', 'pcm_s24le', '-f', 'framemd5', '-vn', master_metadata_dir + '/audio/' + os.path.basename(master_parent_dir) + '.framemd5']
print seq2prores
subprocess.call(seq2prores,env=env_dict)
representation_uuid = str(uuid.uuid4())
split_list = os.path.basename(mezzanine_parent_dir).split('_')
premisxml, premis_namespace, doc, premis = setup_xml(mezzanine_file)
items = {"workflow":"seq2prores","oe":'n/a', "filmographic":split_list[0], "sourceAccession":split_list[1], "interventions":['placeholder'], "prepList":['placeholder'], "user":user}
premis = doc.getroot()
xml_info = make_premis(mezzanine_file, items, premis, premis_namespace,premisxml, representation_uuid, '????')
sequence = xml_info[3]
linking_representation_uuids = []
linking_representation_uuids.append(xml_info[2])
linking_representation_uuids.append(xml_info[2]) # the duplicate does nothing btw, they are a placeholder from a hardcoded function
linking_representation_uuids.append(source_representation_uuid)
create_representation(premisxml, premis_namespace, doc, premis, items,linking_representation_uuids, representation_uuid,sequence )
doc = xml_info[0]
premisxml = xml_info[1]
final_sip_manifest_uuid = str(uuid.uuid4())
prores_event_uuid = str(uuid.uuid4())
macMiniTelecineMachineAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineMachineAgent = make_agent(premis,macMiniTelecineMachineAgent_events, '230d72da-07e7-4a79-96ca-998b9f7a3e41')
macMiniTelecineMachineOSAgent_events = [prores_event_uuid,final_sip_manifest_uuid ]
macMiniTelecineOSAgent = make_agent(premis,macMiniTelecineMachineOSAgent_events, '9486b779-907c-4cc4-802c-22e07dc1242f')
hashlib_events = [final_sip_manifest_uuid ]
hashlibAgent = make_agent(premis,hashlib_events, '9430725d-7523-4071-9063-e8a6ac4f84c4')
ffmpegAgent_events = [prores_event_uuid ]
ffmpegAgent = make_agent(premis,ffmpegAgent_events , 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
operatorEvents = [final_sip_manifest_uuid,prores_event_uuid]
operatorAgent = make_agent(premis,operatorEvents ,user)
#ffmpegAgent = make_agent(premis,[framemd5_uuid ], 'ee83e19e-cdb1-4d83-91fb-7faf7eff738e')
make_event(premis, 'creation', 'Image Sequence and WAV re-encoded to Apple Pro Res 422 HQ with 48khz 24-bit PCM audio', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, ffmpegAgent, operatorAgent ],prores_event_uuid,[representation_uuid], 'outcome')
print premisxml
mezzanine_mediainfoxml = "%s/%s_mediainfo.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
tracexml = "%s/%s_mediatrace.xml" % (mezzanine_metadata_dir,os.path.basename(mezzanine_parent_dir) )
audio_mediainfoxml = "%s/%s_mediainfo.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
audio_mediatracexml = "%s/%s_mediatrace.xml" % (master_metadata_dir + '/audio',os.path.basename(master_audio) )
if not os.path.isfile(audio_mediainfoxml):
make_mediainfo(audio_mediainfoxml,'audiomediaxmlinput',master_audio)
if not os.path.isfile(audio_mediatracexml):
make_mediainfo(audio_mediatracexml,'audiomediatraceinput',master_audio)
if not os.path.isfile(mezzanine_mediainfoxml):
make_mediainfo(mezzanine_mediainfoxml,'mediaxmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
if not os.path.isfile(tracexml):
make_mediatrace(tracexml,'mediatracexmlinput',mezzanine_object_dir + '/' + os.path.basename(mezzanine_parent_dir) + '_mezzanine.mov')
hashlib_manifest(master_parent_dir, source_manifest, master_parent_dir)
hashlib_manifest(mezzanine_parent_dir, mezzanine_manifest, mezzanine_parent_dir)
make_event(premis, 'message digest calculation', 'Checksum manifest for whole package created', [macMiniTelecineMachineAgent ,macMiniTelecineOSAgent, operatorAgent],final_sip_manifest_uuid,[representation_uuid], 'source')
write_premis(doc, premisxml)
finish = datetime.datetime.now()
append_csv(csv_report_filename, (os.path.basename( master_parent_dir), start, finish))
'''
remove_bad_files(source)
source_count = 0
for root, directories, filenames in os.walk(source):
for files in filenames:
source_count +=1 #works in windows at least
if os.path.isfile(manifest):
count_in_manifest = manifest_file_count(manifest)
if source_count != count_in_manifest:
print 'This manifest may be outdated as the number of files in your directory does not match the number of files in the manifest'
generate_log(log_name_source, 'EVENT = Existing source manifest check - Failure - The number of files in the source directory is not equal to the number of files in the source manifest ')
sys.exit()
source_manifest_start_time = time.time()
if not os.path.isfile(manifest):
try:
print 'Generating source manifest'
hashlib_manifest(source, manifest,source)
generate_log(log_name_source, 'EVENT = Generating source manifest')
shutil.move(log_name_source, source)
except OSError:
print 'You do not have access to this directory. Perhaps it is read only, or the wrong file system\n'
sys.exit()
else:
generate_log(log_name_source, 'EVENT = Existing source manifest check - Source manifest already exists. Script will exit. ')
source_manifest_time = time.time() - source_manifest_start_time
print 'Manifest created in %s' % manifest
def make_ffv1(video_files, csv_report_filename):
for filename in video_files: #loop all files in directory
filenoext = os.path.splitext(filename)[0]
# Generate new directory names
metadata_dir = "%s/metadata" % filenoext
log_dir = "%s/logs" % filenoext
data_dir = "%s/objects" % filenoext
# Actually create the directories.
os.makedirs(metadata_dir)
os.makedirs(data_dir)
os.makedirs(log_dir)
#Generate filenames for new files.
inputxml = "%s/%s_source_mediainfo.xml" % (metadata_dir,
os.path.basename(filename))
inputtracexml = "%s/%s_source_mediatrace.xml" % (
metadata_dir, os.path.basename(filename))
output = "%s/%s.mkv" % (
data_dir, os.path.splitext(os.path.basename(filename))[0])
# Generate filename of ffv1.mkv without the path.
outputfilename = os.path.basename(output)
outputxml = "%s/%s_mediainfo.xml" % (metadata_dir, outputfilename)
outputtracexml = "%s/%s_mediatrace.xml" % (metadata_dir,
outputfilename)
fmd5 = "%s/%s_source.framemd5" % (metadata_dir,
os.path.basename(filename))
fmd5ffv1 = "%s/%s_ffv1.framemd5" % (metadata_dir, outputfilename)
log = "%s/%s_log.log" % (log_dir, filename)
generate_log(log, 'Input = %s' % filename)
generate_log(log, 'Output = %s' % output)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation of source started.'
)
ffv1_logfile = log_dir + '/%s_ffv1_transcode.log' % filename
ffv1_env_dict = set_environment(ffv1_logfile)
par = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%PixelAspectRatio%", filename
]).rstrip()
field_order = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full',
"--Inform=Video;%ScanType%", filename
]).rstrip()
height = subprocess.check_output([
'mediainfo', '--Language=raw', '--Full', "--Inform=Video;%Height%",
filename
]).rstrip()
# Transcode video file writing frame md5 and output appropriately
ffv1_command = [
'ffmpeg',
'-i',
filename,
'-c:v',
'ffv1', # Use FFv1 codec
'-g',
'1', # Use intra-frame only aka ALL-I aka GOP=1
'-level',
'3', # Use Version 3 of FFv1
'-c:a',
'copy', # Copy and paste audio bitsream with no transcoding
'-map',
'0',
'-dn',
'-report',
'-slicecrc',
'1',
'-slices',
'16',
]
# check for FCP7 lack of description and PAL
if par == '1.000':
if field_order == '':
if height == '576':
ffv1_command += ['-vf', 'setfield=tff, setdar=4/3']
ffv1_command += [
output,
'-f',
'framemd5',
'-an', # Create decoded md5 checksums for every frame of the input. -an ignores audio
fmd5
]
print ffv1_command
subprocess.call(ffv1_command, env=ffv1_env_dict)
generate_log(
log,
'makeffv1.py transcode to FFV1 and framemd5 generation completed.')
generate_log(
log, 'makeffv1.py Framemd5 generation of output file started.')
fmd5_logfile = log_dir + '/%s_framemd5.log' % outputfilename
fmd5_env_dict = set_environment(fmd5_logfile)
pix_fmt = get_ffmpeg_fmt(filename, 'video')
fmd5_command = [
'ffmpeg', # Create decoded md5 checksums for every frame
'-i',
output,
'-report',
'-pix_fmt',
pix_fmt,
'-f',
'framemd5',
'-an',
fmd5ffv1
]
print fmd5_command
subprocess.call(fmd5_command, env=fmd5_env_dict)
generate_log(
log, 'makeffv1.py Framemd5 generation of output file completed')
source_video_size = get_mediainfo('source_video_size',
"--inform=General;%FileSize%",
filename)
ffv1_video_size = get_mediainfo('ffv1_video_size',
'--inform=General;%FileSize%', output)
compression_ratio = float(source_video_size) / float(ffv1_video_size)
if os.path.basename(sys.argv[0]) == 'makeffv1.py':
try:
shutil.copy(sys.argv[0], log_dir)
except IOError:
pass
print 'Generating mediainfo xml of input file and saving it in %s' % inputxml
make_mediainfo(inputxml, 'mediaxmlinput', filename)
print 'Generating mediainfo xml of output file and saving it in %s' % outputxml
make_mediainfo(outputxml, 'mediaxmloutput', output)
print 'Generating mediatrace xml of input file and saving it in %s' % inputtracexml
make_mediatrace(inputtracexml, 'mediatracexmlinput', filename)
print 'Generating mediatrace xml of output file and saving it in %s' % outputtracexml
make_mediatrace(outputtracexml, 'mediatracexmloutput', output)
source_parent_dir = os.path.dirname(os.path.abspath(filename))
manifest = '%s/%s_manifest.md5' % (source_parent_dir, filenoext)
generate_log(log, 'makeffv1.py MD5 manifest started')
checksum_mismatches = []
with open(fmd5) as f1:
with open(fmd5ffv1) as f2:
for (lineno1, line1), (lineno2, line2) in itertools.izip(
read_non_comment_lines(f1),
read_non_comment_lines(f2)):
if line1 != line2:
if 'sar' in line1:
checksum_mismatches = ['sar']
else:
checksum_mismatches.append(1)
if len(checksum_mismatches) == 0:
print 'LOSSLESS'
append_csv(csv_report_filename,
(output, 'LOSSLESS', source_video_size, ffv1_video_size,
compression_ratio))
generate_log(log, 'makeffv1.py Transcode was lossless')
elif len(checksum_mismatches) == 1:
if checksum_mismatches[0] == 'sar':
print 'Image content is lossless,'
' Pixel Aspect Ratio has been altered.'
' Update ffmpeg in order to resolve the PAR issue.'
append_csv(
csv_report_filename,
(output, 'LOSSLESS - different PAR', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(
log,
'makeffv1.py Image content is lossless but Pixel Aspect Ratio has been altered.Update ffmpeg in order to resolve the PAR issue.'
)
elif len(checksum_mismatches) > 1:
print 'NOT LOSSLESS'
append_csv(csv_report_filename,
(output, 'NOT LOSSLESS', source_video_size,
ffv1_video_size, compression_ratio))
generate_log(log, 'makeffv1.py Not Lossless.')
hashlib_manifest(filenoext, manifest, source_parent_dir)
if filecmp.cmp(fmd5, fmd5ffv1, shallow=False):
print "YOUR FILES ARE LOSSLESS YOU SHOULD BE SO HAPPY!!!"
else:
print "The framemd5 text files are not completely identical."
" This may be because of a lossy transcode,"
" or a change in metadata, most likely pixel aspect ratio."
" Please analyse the framemd5 files for source and output."
for root, directories, filenames in os.walk(source):
for files in filenames:
source_count +=1 #works in windows at least
if os.path.isfile(manifest):
count_in_manifest = manifest_file_count(manifest)
if source_count != count_in_manifest:
print 'This manifest may be outdated as the number of files in your directory does not match the number of files in the manifest'
generate_log(log_name_source, 'EVENT = Existing source manifest check - Failure - The number of files in the source directory is not equal to the number of files in the source manifest ')
sys.exit()
source_manifest_start_time = time.time()
if not os.path.isfile(manifest):
try:
print 'Generating source manifest'
if args.f:
hashlib_manifest(source, manifest,source)
else:
hashlib_manifest(source, manifest,source_parent_dir)
generate_log(log_name_source, 'EVENT = Generating source manifest')
shutil.move(log_name_source, source)
except OSError:
print 'You do not have access to this directory. Perhaps it is read only, or the wrong file system\n'
sys.exit()
else:
generate_log(log_name_source, 'EVENT = Existing source manifest check - Source manifest already exists. Script will exit. ')
source_manifest_time = time.time() - source_manifest_start_time
print 'Manifest created in %s' % manifest
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
clairmeta_version = clairmeta.__version__
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print(args)
user = ififuncs.determine_user(args)
object_entry = get_object_entry(args)
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
uuid, uuid_event = determine_uuid(args, sip_path)
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
if args.d:
content_title = create_content_title_text(sip_path, args)
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
ififuncs.get_technical_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print(('\n', user,
'ran this script at %s and it finished at %s' % (start, finish)))
if args.d:
process_dcp(sip_path, content_title, args, new_manifest_textfile,
new_log_textfile, metadata_dir, clairmeta_version)
return new_log_textfile, new_manifest_textfile
