def process_dcp(sip_path, content_title, args, new_manifest_textfile, new_log_textfile, metadata_dir, clairmeta_version): ''' Runs DCP specific functions. ''' objects_dir = os.path.join(sip_path, 'objects') cpl = ififuncs.find_cpl(objects_dir) dcp_dirname = os.path.dirname(cpl) os.chdir(os.path.dirname(dcp_dirname)) os.rename(os.path.basename(dcp_dirname), content_title) new_dcp_path = os.path.join('objects', content_title).replace("\\", "/") absolute_dcp_path = os.path.join(sip_path, new_dcp_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), new_dcp_path) ''' a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT) b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT) ''' dcp = DCP(absolute_dcp_path) dcp_dict = dcp.parse() # json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': ')) xml_str = dicttoxml.dicttoxml(dcp_dict, custom_root='ClairmetaProbe', ids=False, attr_type=False) xml_pretty = prettyprint_xml(xml_str) status, report = dcp.check() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s' % (status, report, clairmeta_version)) clairmeta_xml = os.path.join(metadata_dir, '%s_clairmeta.xml' % content_title) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s' % (clairmeta_xml, clairmeta_version)) with open(clairmeta_xml, 'w') as fo: fo.write(xml_pretty) ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile, 'md5') ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml) print(status) print(report)
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i print args if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started') ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')) ififuncs.generate_log(new_log_textfile, 'Command line arguments: %s' % args) ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user) ififuncs.generate_log(new_log_textfile, uuid_event) if args.u is False: sys.exit() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry) metadata_dir = os.path.join(sip_path, 'metadata') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest(metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append(logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir))) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(args, sip_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), os.path.join('objects', content_title).replace("\\", "/")) return new_log_textfile, new_manifest_textfile
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i print args if args.user: user = args.user else: user = ififuncs.get_user() if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) != 4: print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid ) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log( new_log_textfile, 'EVENT = sipcreator.py started' ) ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py') ) ififuncs.generate_log( new_log_textfile, 'Command line arguments: %s' % args ) ififuncs.generate_log( new_log_textfile, 'EVENT = agentName=%s' % user ) ififuncs.generate_log( new_log_textfile, uuid_event ) if args.u is False: sys.exit() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry ) metadata_dir = os.path.join(sip_path, 'metadata') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest( metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir ) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append( logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir)) ) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(args, sip_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), os.path.join('objects', content_title).replace("\\", "/") ) return new_log_textfile, new_manifest_textfile
def main(args_): ''' Launch all the functions for creating an IFI SIP. ''' args = parse_args(args_) start = datetime.datetime.now() inputs = args.i if args.d: try: import clairmeta except ImportError: print( 'Exiting as Clairmeta is not installed. If there is a case for not using clairmeta, please let me know and i can make a workaround' ) sys.exit() print args if args.user: user = args.user else: user = ififuncs.get_user() if not args.sc: if args.oe: if args.oe[:2] != 'oe': print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif len(args.oe[2:]) not in range(4, 6): print 'First two characters must be \'oe\' and last four characters must be four digits' object_entry = ififuncs.get_object_entry() elif not args.oe[2:].isdigit(): object_entry = ififuncs.get_object_entry() print 'First two characters must be \'oe\' and last four characters must be four digits' else: object_entry = args.oe else: object_entry = ififuncs.get_object_entry() else: object_entry = 'not_applicable' sip_path = make_folder_path(os.path.join(args.o), args, object_entry) if args.u: if ififuncs.validate_uuid4(args.u) is None: uuid = args.u uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4' ) % uuid else: print 'exiting due to invalid UUID' uuid_event = ( 'EVENT = exiting due to invalid UUID supplied on the commmand line: %s' % uuid) uuid = False else: uuid = os.path.basename(sip_path) uuid_event = ( 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=UUID, value=%s, module=uuid.uuid4') % uuid new_log_textfile = os.path.join(sip_path, 'logs' + '/' + uuid + '_sip_log.log') ififuncs.generate_log(new_log_textfile, 'EVENT = sipcreator.py started') ififuncs.generate_log( new_log_textfile, 'eventDetail=sipcreator.py %s' % ififuncs.get_script_version('sipcreator.py')) ififuncs.generate_log(new_log_textfile, 'Command line arguments: %s' % args) ififuncs.generate_log(new_log_textfile, 'EVENT = agentName=%s' % user) ififuncs.generate_log(new_log_textfile, uuid_event) if args.u is False: sys.exit() if not args.sc: ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=Identifier assignement,' ' eventIdentifierType=object entry, value=%s' % object_entry) metadata_dir = os.path.join(sip_path, 'metadata') supplemental_dir = os.path.join(metadata_dir, 'supplemental') logs_dir = os.path.join(sip_path, 'logs') log_names = move_files(inputs, sip_path, args) get_metadata(sip_path, new_log_textfile) ififuncs.hashlib_manifest(metadata_dir, metadata_dir + '/metadata_manifest.md5', metadata_dir) if args.sc: normalise_objects_manifest(sip_path) new_manifest_textfile = consolidate_manifests(sip_path, 'objects', new_log_textfile) consolidate_manifests(sip_path, 'metadata', new_log_textfile) ififuncs.hashlib_append(logs_dir, new_manifest_textfile, os.path.dirname(os.path.dirname(logs_dir))) if args.supplement: os.makedirs(supplemental_dir) supplement_cmd = [ '-i', args.supplement, '-user', user, '-new_folder', supplemental_dir, os.path.dirname(sip_path), '-copy' ] package_update.main(supplement_cmd) if args.sc: print('Generating Digital Forensics XML') dfxml = accession.make_dfxml(args, sip_path, uuid) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=File system metadata extraction using Digital Forensics XML, eventOutcome=%s, agentName=makedfxml' % (dfxml)) ififuncs.manifest_update(new_manifest_textfile, dfxml) sha512_log = manifest.main([sip_path, '-sha512', '-s']) sha512_manifest = os.path.join(os.path.dirname(sip_path), uuid + '_manifest-sha512.txt') ififuncs.merge_logs_append(sha512_log, new_log_textfile, new_manifest_textfile) ififuncs.checksum_replace(sha512_manifest, new_log_textfile, 'sha512') os.remove(sha512_log) ififuncs.sort_manifest(new_manifest_textfile) if not args.quiet: log_report(log_names) finish = datetime.datetime.now() print '\n', user, 'ran this script at %s and it finished at %s' % (start, finish) if args.d: content_title = create_content_title_text(sip_path) new_dcp_path = os.path.join('objects', content_title).replace("\\", "/") absolute_dcp_path = os.path.join(sip_path, new_dcp_path) ififuncs.manifest_replace( new_manifest_textfile, os.path.join('objects', os.path.basename(args.i[0])).replace("\\", "/"), new_dcp_path) ''' a = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'check', '-type', 'dcp', absolute_dcp_path], stderr=subprocess.STDOUT) b = subprocess.check_output(['python', '-m', 'clairmeta.cli', 'probe', '-type', 'dcp', '-format', 'xml', absolute_dcp_path], stderr=subprocess.STDOUT) ''' dcp = DCP(absolute_dcp_path) clairmeta_version = clairmeta.__version__ dcp_dict = dcp.parse() # json_str = json.dumps(dcp_dict , sort_keys=True, indent=2, separators=(',', ': ')) xml_str = dicttoxml.dicttoxml(dcp_dict, custom_root='ClairmetaProbe', ids=False, attr_type=False) xml_pretty = prettyprint_xml(xml_str) status, report = dcp.check() ififuncs.generate_log( new_log_textfile, 'EVENT = eventType=validation, eventOutcome=%s, eventDetail=%s, agentName=Clairmeta version %s' % (status, report, clairmeta_version)) clairmeta_xml = os.path.join(metadata_dir, '%s_clairmeta.xml' % content_title) ififuncs.generate_log( new_log_textfile, 'EVENT = Metadata extraction - eventDetail=Clairmeta DCP metadata extraction, eventOutcome=%s, agentName=Clairmeta version %s' % (clairmeta_xml, clairmeta_version)) with open(clairmeta_xml, 'w') as fo: fo.write(xml_pretty) ififuncs.checksum_replace(new_manifest_textfile, new_log_textfile, 'md5') ififuncs.manifest_update(new_manifest_textfile, clairmeta_xml) print status print report print '\n', user, 'ran this script at %s and it finished at %s' % ( start, finish) return new_log_textfile, new_manifest_textfile