def cli_check(args):
try:
if args.type == 'dcp':
check_profile = DCP_CHECK_PROFILE
callback = None
if args.profile:
path = os.path.abspath(args.profile)
check_profile = load_profile(path)
if args.log:
check_profile['log_level'] = args.log
if args.progress:
callback = console_progress_bar
status, _ = DCP(args.path, kdm=args.kdm, pkey=args.key).check(
profile=check_profile, ov_path=args.ov, hash_callback=callback)
else:
obj_type = package_type_map[args.type]
setting = package_check_settings[args.type]
status = obj_type(args.path).check(setting)
except Exception as e:
status = False
print("Error : {}".format(e))
msg = "{} - {} - Check {}".format(
args.type.upper(), args.path, "succeeded" if status else "failed")
return status, msg
def process_dcp(sip_path, content_title, args, new_manifest_textfile,
new_log_textfile, metadata_dir, clairmeta_version):
'''
Runs DCP specific functions.
'''
objects_dir = os.path.join(sip_path, 'objects')
cpl = ififuncs.find_cpl(objects_dir)
dcp_dirname = os.path.dirname(cpl)
os.chdir(os.path.dirname(dcp_dirname))
os.rename(os.path.basename(dcp_dirname), content_title)
new_dcp_path = os.path.join('objects', content_title).replace("\\", "/")
absolute_dcp_path = os.path.join(sip_path, new_dcp_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
new_dcp_path)
'''
a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT)
b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT)
'''
dcp = DCP(absolute_dcp_path)
dcp_dict = dcp.parse()
# json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': '))
xml_str = dicttoxml.dicttoxml(dcp_dict,
custom_root='ClairmetaProbe',
ids=False,
attr_type=False)
xml_pretty = prettyprint_xml(xml_str)
status, report = dcp.check()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s'
% (status, report, clairmeta_version))
clairmeta_xml = os.path.join(metadata_dir,
'%s_clairmeta.xml' % content_title)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s'
% (clairmeta_xml, clairmeta_version))
with open(clairmeta_xml, 'w') as fo:
fo.write(xml_pretty)
ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile, 'md5')
ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml)
print(status)
print(report)
def cli_copy(args):
dcp = DCP(args.source)
dcp_size = dcp.size
try:
log = get_log()
log.info("Copy {} to {}".format(args.source, args.dest))
start = time.time()
progress = ConsoleProgress()
progress._total_size = dcp_size
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(
shutil.copytree,
args.source,
args.dest,
dirs_exist_ok=args.overwrite
)
while future.running():
copy_size = folder_size(args.dest)
elapsed = time.time() - start
progress(args.source, copy_size, dcp_size, elapsed)
time.sleep(1.0)
future.result()
progress(args.source, dcp_size, dcp_size, elapsed)
log.info("Total time : {:.2f} sec".format(time.time() - start))
dcp_dst = DCP(args.dest)
status, report = dcp.check(hash_callback=ConsoleProgress())
return status
except Exception as e:
print(str(e))
return False
def cli_check(args):
try:
if args.type == 'dcp':
check_profile = DCP_CHECK_PROFILE
callback = None
if args.profile:
path = os.path.abspath(args.profile)
check_profile = load_profile(path)
if args.log:
check_profile['log_level'] = args.log
if args.progress:
callback = ConsoleProgress()
if args.format != 'text':
disable_log()
status, report = DCP(args.path, kdm=args.kdm,
pkey=args.key).check(profile=check_profile,
ov_path=args.ov,
hash_callback=callback)
if args.format == "dict":
msg = pprint.pformat(report.to_dict())
elif args.format == "json":
msg = json.dumps(report.to_dict(),
sort_keys=True,
indent=2,
separators=(',', ': '))
elif args.format == "xml":
xml_str = dicttoxml.dicttoxml(report.to_dict(),
custom_root='ClairmetaCheck',
ids=False,
attr_type=False)
msg = prettyprint_xml(xml_str)
if args.format != 'text':
return True, msg
else:
obj_type = package_type_map[args.type]
setting = package_check_settings[args.type]
status = obj_type(args.path).check(setting)
except Exception as e:
status = False
print("Error : {}".format(e))
msg = "{} - {} - Check {}".format(args.type.upper(), args.path,
"succeeded" if status else "failed")
return status, msg
def cli_check(args):
try:
if args.type == 'dcp':
check_profile = DCP_CHECK_PROFILE
if args.profile:
path = os.path.abspath(args.profile)
check_profile = load_profile(path)
if args.log:
check_profile['log_level'] = args.log
status, _ = DCP(args.path).check(
profile=check_profile, ov_path=args.ov)
else:
obj_type = package_type_map[args.type]
status = obj_type(args.path).check()
return status, ""
except Exception as e:
return False, "Error : {}".format(e)
def cli_check(args):
try:
if args.type == 'dcp':
check_profile = DCP_CHECK_PROFILE
if args.profile:
path = os.path.abspath(args.profile)
check_profile = load_profile(path)
if args.log:
check_profile['log_level'] = args.log
status, _ = DCP(args.path).check(profile=check_profile,
ov_path=args.ov)
else:
obj_type = package_type_map[args.type]
status = obj_type(args.path).check()
return status
except Exception as e:
print('Error : ' + str(e), file=sys.stderr)
def main(args_):
'''
Launch all the functions for creating an IFI SIP.
'''
args = parse_args(args_)
start = datetime.datetime.now()
inputs = args.i
if args.d:
try:
import clairmeta
except ImportError:
print(
'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround'
)
sys.exit()
print args
if args.user:
user = args.user
else:
user = ififuncs.get_user()
if not args.sc:
if args.oe:
if args.oe[:2] != 'oe':
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif len(args.oe[2:]) not in range(4, 6):
print 'First two characters must be \'oe\' and last four characters must be four digits'
object_entry = ififuncs.get_object_entry()
elif not args.oe[2:].isdigit():
object_entry = ififuncs.get_object_entry()
print 'First two characters must be \'oe\' and last four characters must be four digits'
else:
object_entry = args.oe
else:
object_entry = ififuncs.get_object_entry()
else:
object_entry = 'not_applicable'
sip_path = make_folder_path(os.path.join(args.o), args, object_entry)
if args.u:
if ififuncs.validate_uuid4(args.u) is None:
uuid = args.u
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4'
) % uuid
else:
print 'exiting due to invalid UUID'
uuid_event = (
'EVENT = exiting due to invalid UUID supplied on the commmand line: %s'
% uuid)
uuid = False
else:
uuid = os.path.basename(sip_path)
uuid_event = (
'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid
new_log_textfile = os.path.join(sip_path,
'logs' + '/' + uuid + '_sip_log.log')
ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started')
ififuncs.generate_log(
new_log_textfile, 'eventDetail=sipcreator.py %s' %
ififuncs.get_script_version('sipcreator.py'))
ififuncs.generate_log(new_log_textfile,
'Command line arguments: %s' % args)
ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user)
ififuncs.generate_log(new_log_textfile, uuid_event)
if args.u is False:
sys.exit()
if not args.sc:
ififuncs.generate_log(
new_log_textfile, 'EVENT = eventType=Identifier assignement,'
' eventIdentifierType=object entry, value=%s' % object_entry)
metadata_dir = os.path.join(sip_path, 'metadata')
supplemental_dir = os.path.join(metadata_dir, 'supplemental')
logs_dir = os.path.join(sip_path, 'logs')
log_names = move_files(inputs, sip_path, args)
get_metadata(sip_path, new_log_textfile)
ififuncs.hashlib_manifest(metadata_dir,
metadata_dir + '/metadata_manifest.md5',
metadata_dir)
if args.sc:
normalise_objects_manifest(sip_path)
new_manifest_textfile = consolidate_manifests(sip_path, 'objects',
new_log_textfile)
consolidate_manifests(sip_path, 'metadata', new_log_textfile)
ififuncs.hashlib_append(logs_dir, new_manifest_textfile,
os.path.dirname(os.path.dirname(logs_dir)))
if args.supplement:
os.makedirs(supplemental_dir)
supplement_cmd = [
'-i', args.supplement, '-user', user, '-new_folder',
supplemental_dir,
os.path.dirname(sip_path), '-copy'
]
package_update.main(supplement_cmd)
if args.sc:
print('Generating Digital Forensics XML')
dfxml = accession.make_dfxml(args, sip_path, uuid)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml'
% (dfxml))
ififuncs.manifest_update(new_manifest_textfile, dfxml)
sha512_log = manifest.main([sip_path, '-sha512', '-s'])
sha512_manifest = os.path.join(os.path.dirname(sip_path),
uuid + '_manifest-sha512.txt')
ififuncs.merge_logs_append(sha512_log, new_log_textfile,
new_manifest_textfile)
ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512')
os.remove(sha512_log)
ififuncs.sort_manifest(new_manifest_textfile)
if not args.quiet:
log_report(log_names)
finish = datetime.datetime.now()
print '\n', user, 'ran this script at %s and it finished at %s' % (start,
finish)
if args.d:
content_title = create_content_title_text(sip_path)
new_dcp_path = os.path.join('objects',
content_title).replace("\\", "/")
absolute_dcp_path = os.path.join(sip_path, new_dcp_path)
ififuncs.manifest_replace(
new_manifest_textfile,
os.path.join('objects',
os.path.basename(args.i[0])).replace("\\", "/"),
new_dcp_path)
'''
a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT)
b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT)
'''
dcp = DCP(absolute_dcp_path)
clairmeta_version = clairmeta.__version__
dcp_dict = dcp.parse()
# json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': '))
xml_str = dicttoxml.dicttoxml(dcp_dict,
custom_root='ClairmetaProbe',
ids=False,
attr_type=False)
xml_pretty = prettyprint_xml(xml_str)
status, report = dcp.check()
ififuncs.generate_log(
new_log_textfile,
'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s'
% (status, report, clairmeta_version))
clairmeta_xml = os.path.join(metadata_dir,
'%s_clairmeta.xml' % content_title)
ififuncs.generate_log(
new_log_textfile,
'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s'
% (clairmeta_xml, clairmeta_version))
with open(clairmeta_xml, 'w') as fo:
fo.write(xml_pretty)
ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile,
'md5')
ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml)
print status
print report
print '\n', user, 'ran this script at %s and it finished at %s' % (
start, finish)
return new_log_textfile, new_manifest_textfile