def main(args_):
'''
Launch all the functions for determining folder and file paths.
'''
args = parse_args(args_)
source = args.input
sip_path = ififuncs.check_for_sip([source])
if sip_path is not None:
package_info = analyse_package(sip_path)
if package_info is False:
print('Valid UUID not found in folder path')
else:
print package_info
error_list, question_list = test_package(package_info)
return package_info, error_list, question_list
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
source = args.input
sip_path = ififuncs.check_for_sip([source])
if sip_path is not None:
oe_path = os.path.dirname(sip_path)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
start = datetime.datetime.now()
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = rearrange.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=rearrange.py %s' %
ififuncs.get_script_version('rearrange.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
if not os.path.isdir(args.new_folder):
os.makedirs(args.new_folder)
for filename in args.i:
# add test to see if it actually deleted - what if read only?
shutil.move(filename, args.new_folder)
print '%s has been moved into %s' % (filename, args.new_folder)
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=file movement,'
' eventOutcomeDetailNote=%s has been moved into %s'
' agentName=shutil.move()' % (filename, args.new_folder))
relative_filename = filename.replace(args.input + '/', '')
relative_new_folder = args.new_folder.replace(args.input + '/', '')
update_manifest(
sip_manifest, relative_filename,
os.path.join(relative_new_folder,
os.path.basename(relative_filename)),
new_log_textfile)
ififuncs.generate_log(new_log_textfile, 'EVENT = rearrange.py finished')
ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
source = args.uuid_path
sip_path = ififuncs.check_for_sip([source])
if sip_path is not None:
oe_path = os.path.dirname(sip_path)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
else:
# this is assuming that the other workflow will be the
# special collections workflow that has the uuid as the parent.
# some real checks should exist for this whole if/else flow.
sip_path = args.uuid_path
oe_path = os.path.dirname(args.uuid_path)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(oe_path, uuid + '_manifest.md5')
start = datetime.datetime.now()
print(args)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = deletefiles.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=deletefiles.py %s' %
ififuncs.get_script_version('deletefiles.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
metadata_dir = os.path.join(sip_path, 'metadata')
for filename in args.i:
# add test to see if it actually deleted - what if read only?
os.remove(filename)
print(('%s has been deleted' % filename))
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=deletion,'
' eventOutcomeDetailNote=%s has been deleted,'
' agentName=os.remove()' % filename)
for metadata in os.listdir(metadata_dir):
if os.path.basename(filename) in metadata:
os.remove(os.path.join(metadata_dir, metadata))
print(('%s has been deleted' %
os.path.join(metadata_dir, metadata)))
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=deletion,'
' eventOutcomeDetailNote=%s has been deleted,'
' agentName=os.remove()' %
os.path.join(metadata_dir, metadata))
remove_from_manifest(sip_manifest, os.path.basename(filename),
new_log_textfile)
ififuncs.sort_manifest(sip_manifest)
ififuncs.generate_log(new_log_textfile, 'EVENT = deletefiles.py finished')
ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
finish = datetime.datetime.now()
print('\n- %s ran this script at %s and it finished at %s' %
(user, start, finish))
def main(args_):
'''
Launches the various functions that will accession a package
'''
args = parse_args(args_)
source = args.input
uuid_directory = ififuncs.check_for_sip([source])
if uuid_directory is not None:
oe_path = os.path.dirname(uuid_directory)
oe_number = os.path.basename(oe_path)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.number:
if args.number[:3] != 'aaa':
print 'First three characters must be \'aaa\' and last four characters must be four digits'
accession_number = ififuncs.get_accession_number()
elif len(args.number[3:]) != 4:
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
elif not args.number[3:].isdigit():
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
else:
accession_number = args.number
else:
accession_number = ififuncs.get_accession_number()
if args.pbcore:
if args.reference:
Reference_Number = args.reference.upper()
else:
Reference_Number = ififuncs.get_reference_number()
if args.acquisition_type:
acquisition_type = ififuncs.get_acquisition_type(
args.acquisition_type)
print acquisition_type
accession_path = os.path.join(os.path.dirname(oe_path),
accession_number)
uuid = os.path.basename(uuid_directory)
new_uuid_path = os.path.join(accession_path, uuid)
logs_dir = os.path.join(new_uuid_path, 'logs')
sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
if args.force:
proceed = 'Y'
else:
proceed = ififuncs.ask_yes_no('Do you want to rename %s with %s' %
(oe_number, accession_number))
if proceed == 'Y':
os.rename(oe_path, accession_path)
if args.register:
register = args.register
else:
register = make_register()
ififuncs.append_csv(register,
(oe_number.upper()[:2] + '-' + oe_number[2:],
accession_number, '', '', '', '', '', ''))
ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py started')
ififuncs.generate_log(
sipcreator_log, 'eventDetail=accession.py %s' %
ififuncs.get_script_version('accession.py'))
ififuncs.generate_log(sipcreator_log,
'Command line arguments: %s' % args)
ififuncs.generate_log(sipcreator_log, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(
sipcreator_log, 'EVENT = eventType=Identifier assignment,'
' eventIdentifierType=accession number, value=%s' %
accession_number)
ififuncs.generate_log(
sipcreator_log, 'EVENT = eventType=accession,'
' eventIdentifierType=accession number, value=%s' %
accession_number)
sip_manifest = os.path.join(accession_path, uuid) + '_manifest.md5'
sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(new_uuid_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
os.remove(sha512_log)
print('Generating Digital Forensics XML')
dfxml = make_dfxml(args, new_uuid_path, uuid)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
# this is inefficient. The script should not have to ask for reference
# number twice if someone wants to insert the filmographic but do not
# want to make the pbcore csv, perhaps because the latter already exists.
if args.csv:
metadata_dir = os.path.join(new_uuid_path, 'metadata')
package_filmographic = os.path.join(
metadata_dir, Reference_Number + '_filmographic.csv')
insert_filmographic(args.csv, Reference_Number,
package_filmographic)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=Filmographic descriptive metadata added to metadata folder, eventOutcome=%s, agentName=accession.py'
% (package_filmographic))
ififuncs.manifest_update(sip_manifest, package_filmographic)
ififuncs.sha512_update(sha512_manifest, package_filmographic)
print('Filmographic descriptive metadata added to metadata folder')
ififuncs.generate_log(sipcreator_log, 'EVENT = accession.py finished')
ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(sip_manifest, dfxml)
ififuncs.sha512_update(sha512_manifest, dfxml)
if args.pbcore:
makepbcore_cmd = [
accession_path, '-p', '-user', user, '-reference',
Reference_Number
]
if args.parent:
makepbcore_cmd.extend(['-parent', args.parent])
if args.acquisition_type:
makepbcore_cmd.extend(
['-acquisition_type', args.acquisition_type])
if args.donor:
makepbcore_cmd.extend(['-donor', args.donor])
if args.donor:
makepbcore_cmd.extend(
['-depositor_reference', args.depositor_reference])
if args.donation_date:
makepbcore_cmd.extend(['-donation_date', args.donation_date])
makepbcore.main(makepbcore_cmd)
else:
print 'not a valid package. The input should include a package that has been through Object Entry'
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
source = args.input
sip_path = ififuncs.check_for_sip([source])
if sip_path is not None:
oe_path = os.path.dirname(sip_path)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(oe_path, uuid) + '_manifest.md5'
else:
# this is assuming that the other workflow will be the
# special collections workflow that has the uuid as the parent.
# some real checks should exist for this whole if/else flow.
sip_path = args.input
oe_path = os.path.dirname(args.input)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(oe_path, uuid + '_manifest.md5')
start = datetime.datetime.now()
print(args)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile,
'EVENT = package_update.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=package_update.py %s' %
ififuncs.get_script_version('package_update.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
if not os.path.isdir(args.new_folder):
os.makedirs(args.new_folder)
if isinstance(args.i[0], (list, )):
args.i = args.i[0]
for filenames in args.i:
if args.copy:
copyit.main([filenames, args.new_folder])
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=file movement,'
' eventOutcomeDetailNote=%s has been moved into %s'
' agentName=copyit.py' % (filenames, args.new_folder))
# this is hardcoded - pick this apart so that any folder can be added to.
# this must be fixed in normalise.py as well.
relative_new_path = args.new_folder.replace(sip_path, '')
print((relative_new_path, 'relative'))
if (relative_new_path[0] == '/') or relative_new_path[0] == '\\':
relative_new_path = relative_new_path[1:].replace('\\', '/')
sipcreator.consolidate_manifests(sip_path, relative_new_path,
new_log_textfile)
log_manifest = os.path.join(
os.path.dirname(new_log_textfile),
os.path.basename(filenames) + '_manifest.md5')
ififuncs.manifest_update(sip_manifest, log_manifest)
ififuncs.sort_manifest(sip_manifest)
else:
# add test to see if it actually deleted - what if read only?
shutil.move(filenames, args.new_folder)
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=file movement,'
' eventOutcomeDetailNote=%s has been moved into %s'
' agentName=shutil.move()' % (filenames, args.new_folder))
print(('%s has been moved into %s' % (filenames, args.new_folder)))
relative_filename = filenames.replace(
os.path.dirname(args.input) + '/', '').replace('\\', '/')
relative_filename = filenames.replace(
os.path.dirname(args.input) + '\\', '').replace('\\', '/')
relative_new_folder = args.new_folder.replace(
os.path.dirname(args.input) + '/', '').replace('\\', '/')
relative_new_folder = args.new_folder.replace(
os.path.dirname(args.input) + '\\', '').replace('\\', '/')
update_manifest(
sip_manifest, relative_filename,
os.path.join(relative_new_folder,
os.path.basename(relative_filename)).replace(
'\\', '/'), new_log_textfile)
ififuncs.generate_log(new_log_textfile,
'EVENT = package_update.py finished')
ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
finish = datetime.datetime.now()
print('\n- %s ran this script at %s and it finished at %s' %
(user, start, finish))
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
source = args.input
sip_path = ififuncs.check_for_sip([source])
if sip_path is not None:
oe_path = os.path.dirname(sip_path)
uuid = os.path.basename(sip_path)
sip_manifest = os.path.join(
oe_path, uuid
) + '_manifest.md5'
start = datetime.datetime.now()
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(
new_log_textfile,
'EVENT = deletefiles.py started'
)
ififuncs.generate_log(
new_log_textfile,
'eventDetail=deletefiles.py %s' % ififuncs.get_script_version('deletefiles.py')
)
ififuncs.generate_log(
new_log_textfile,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
new_log_textfile,
'EVENT = agentName=%s' % user
)
metadata_dir = os.path.join(sip_path, 'metadata')
for filename in args.i:
# add test to see if it actually deleted - what if read only?
os.remove(filename)
print '%s has been deleted' % filename
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=deletion,'
' eventOutcomeDetailNote=%s has been deleted,'
' agentName=os.remove()'
% filename
)
for metadata in os.listdir(metadata_dir):
if os.path.basename(filename) in metadata:
os.remove(os.path.join(metadata_dir, metadata))
print '%s has been deleted' % os.path.join(metadata_dir, metadata)
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=deletion,'
' eventOutcomeDetailNote=%s has been deleted,'
' agentName=os.remove()'
% os.path.join(metadata_dir, metadata)
)
remove_from_manifest(sip_manifest, os.path.basename(filename), new_log_textfile)
ififuncs.generate_log(
new_log_textfile,
'EVENT = deletefiles.py finished'
)
ififuncs.checksum_replace(sip_manifest, new_log_textfile, 'md5')
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish)
def main(args_):
'''
Launches the various functions that will accession a package
'''
args = parse_args(args_)
input = args.input
uuid_directory = ififuncs.check_for_sip([input])
if uuid_directory is not None:
oe_path = os.path.dirname(uuid_directory)
oe_number = os.path.basename(oe_path)
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if args.number:
if args.number[:3] != 'aaa':
print 'First three characters must be \'aaa\' and last four characters must be four digits'
accession_number = ififuncs.get_accession_number()
elif len(args.number[3:]) != 4:
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
elif not args.number[3:].isdigit():
accession_number = ififuncs.get_accession_number()
print 'First three characters must be \'aaa\' and last four characters must be four digits'
else:
accession_number = args.number
else:
accession_number = ififuncs.get_accession_number()
accession_path = os.path.join(
os.path.dirname(oe_path), accession_number
)
uuid = os.path.basename(uuid_directory)
new_uuid_path = os.path.join(accession_path, uuid)
logs_dir = os.path.join(new_uuid_path, 'logs')
sipcreator_log = os.path.join(logs_dir, uuid) + '_sip_log.log'
if args.force:
proceed = 'Y'
else:
proceed = ififuncs.ask_yes_no(
'Do you want to rename %s with %s' % (oe_number, accession_number)
)
if proceed == 'Y':
os.rename(oe_path, accession_path)
register = make_register()
ififuncs.append_csv(register, (oe_number.upper()[:2] + '-' + oe_number[2:6], accession_number, '','','','', ''))
ififuncs.generate_log(
sipcreator_log,
'EVENT = accession.py started'
)
ififuncs.generate_log(
sipcreator_log,
'eventDetail=accession.py %s' % ififuncs.get_script_version('accession.py')
)
ififuncs.generate_log(
sipcreator_log,
'Command line arguments: %s' % args
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = agentName=%s' % user
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = eventType=Identifier assignment,'
' eventIdentifierType=accession number, value=%s'
% accession_number
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = eventType=accession,'
' eventIdentifierType=accession number, value=%s'
% accession_number
)
sip_manifest = os.path.join(
accession_path, uuid
) + '_manifest.md5'
sha512_log = manifest.main([new_uuid_path, '-sha512', '-s'])
sha512_manifest = os.path.join(
os.path.dirname(new_uuid_path), uuid + '_manifest-sha512.txt'
)
ififuncs.merge_logs_append(sha512_log, sipcreator_log, sip_manifest)
os.remove(sha512_log)
dfxml = make_dfxml(args, new_uuid_path, uuid)
ififuncs.generate_log(
sipcreator_log,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)
)
ififuncs.generate_log(
sipcreator_log,
'EVENT = accession.py finished'
)
ififuncs.checksum_replace(sip_manifest, sipcreator_log, 'md5')
ififuncs.checksum_replace(sha512_manifest, sipcreator_log, 'sha512')
ififuncs.manifest_update(sip_manifest, dfxml)
ififuncs.sha512_update(sha512_manifest, dfxml)
if args.pbcore:
makepbcore.main([accession_path, '-p', '-user', user])
else:
print 'not a valid package. The input should include a package that has been through Object Entry'
