def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser( description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by " + __author__ + " " + __email__, ) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( "-b", "--verbosity", help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs="?", const="INFO" ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument("-d", "--debugging", help="set debugging logging", action="store_true") # optionally save the log to a file. # Set a location or use the default constant parser.add_argument("-l", "--log_loc", help="save logging to a file", dest="log_loc") parser.add_argument("dest_root", help="Enter the destination root path", action="store") parser.add_argument( "containing_folder", help="The name of the containing folder on disk " + "(prefix+number)", action="store" ) parser.add_argument( "--rehash", help="Disregard any existing previously generated " + "hashes, recreate them on this run", action="store_true", ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"]: logger.critical( "You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'" ) return 1 if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return 1 # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: logger.info("BEGINS") # Begin module code # validation = ValidateBase(args.dest_root) if validation[0] == True: stageRoot = join(*validation[1:]) else: logger.critical("ENDS: Your staging root appears to not be valid!") exit(1) destinationAdminRoot = join(stageRoot, "admin/") destinationDataRoot = join(stageRoot, "data/") containing_folder = args.containing_folder destinationAdminFolder = join(destinationAdminRoot, containing_folder) destinationDataFolder = join(destinationDataRoot, containing_folder) stagingDebugLog = DebugFileHandler(join(destinationAdminFolder, "log.txt")) logger.addHandler(stagingDebugLog) logger.debug("Creating batch from moved files.") movedFiles = Batch(destinationDataFolder, directory=destinationDataFolder) logger.info("Hashing copied files.") existingHashes = None if args.rehash: logger.info("Rehash argument passed. Not reading existing hashes.") if not args.rehash and exists(join(destinationAdminFolder, "fixityOnDisk.txt")): existingHashes = ReadExistingFixityLog(join(destinationAdminFolder, "fixityOnDisk.txt")) WriteFixityLog(join(destinationAdminFolder, "fixityOnDisk.txt"), movedFiles, existingHashes=existingHashes) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception (" + str(e) + ")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "dest_root", help="Enter the destination root path", action='store' ) parser.add_argument( "containing_folder", help="The name of the containing folder on disk " + "(prefix+number)", action='store' ) parser.add_argument( "--rehash", help="Disregard any existing previously generated " + " hashes, recreate them on this run", action="store_true" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: logger.info("BEGINS") # Begin module code # validation = ValidateBase(args.dest_root) if validation[0] != True: logger.critical("Your staging root isn't valid!") exit(1) else: stageRoot = join(*validation[1:]) destinationAdminRoot = join(stageRoot, 'admin/') containing_folder = args.containing_folder destinationAdminFolder = join(destinationAdminRoot, containing_folder) stagingDebugLog = DebugFileHandler( join(destinationAdminFolder, 'log.txt') ) logger.addHandler(stagingDebugLog) existingOriginalFileHashes = ReadExistingFixityLog( join( destinationAdminFolder, 'fixityFromOrigin.txt' ) ) existingMovedFileHashes = ReadExistingFixityLog( join( destinationAdminFolder, 'fixityOnDisk.txt' ) ) notMoved = [key for key in existingOriginalFileHashes if key not in existingMovedFileHashes] foreignFiles = [key for key in existingMovedFileHashes if key not in existingOriginalFileHashes] badHash = [key for key in existingOriginalFileHashes if key not in notMoved and existingOriginalFileHashes[key] != existingMovedFileHashes[key]] for entry in existingOriginalFileHashes: if entry not in notMoved and entry not in badHash: logger.debug("GOOD: "+entry+":" + str(existingOriginalFileHashes[entry])) elif entry in notMoved: logger.debug("NOT MOVED: "+entry+":" + str(existingOriginalFileHashes[entry])) elif entry in badHash: logger.debug("BAD HASH: "+entry+":" + str(existingOriginalFileHashes[entry])) for entry in foreignFiles: logger.debug("FOREIGN FILE: "+entry) logger.info(str(len(existingMovedFileHashes)) + " file(s) total in the staging area.") logger.info(str(len(notMoved)) + " file(s) not copied.") logger.info(str(len(badHash)) + " file(s) have a different hash from the origin.") logger.info(str(len(foreignFiles)) + " file(s) appear to not have come from the origin.") # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument( "-v", help="See the version of this program", action="version", version=__version__ ) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: # Begin module code # b = Batch(args.root, args.item) for item in b.find_items(from_directory=True): print(item.get_file_path()) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: logger.info("BEGINS") # Begin module code # logger.info("Validating Base Structure.") validation = ValidateBase(args.item) if validation[0] != True: logger.critical("Your staging base has not validated!") logger.critical(validation) exit(1) dataPath=join(*validation[1:], 'data') adminPath=join(*validation[1:], 'admin') logger.info("Checking data directory.") dataValid = ValidateOrganization(dataPath) if dataValid[0] != True: logger.critical("Your data directory is not well formed!") logger.critical(dataValid) exit(1) for x in dataValid[1]['notDirs']: logger.warn("The following appears in the data dir " + "but is not a directory: "+x) logger.info("Checking admin directory.") adminValid = ValidateOrganization(adminPath, reqTopFiles=['record.json', 'fileConversions.txt'], reqDirContents=[ 'fixityFromOrigin.txt', 'fixityOnDisk.txt', 'log.txt', 'rsyncFromOrigin.txt'] ) if adminValid[0] != True: logger.critical("ENDS: Your admin directory is not well formed!") logger.warn(adminValid) exit(1) if dataValid[1]['dirs'] != adminValid[1]['dirs']: for x in dataValid[1]['dirs']: if x not in adminValid[1]['dirs']: logger.warn("Directory appears in data but not admin: "+x) for x in adminValid[1]['dirs']: if x not in dataValid[1]['dirs']: logger.warn("Directory appears in admin but not data: "+x) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument( "-v", help="See the version of this program", action="version", version=__version__ ) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) parser.add_argument( "dest_root", help="Enter the destination root path", action='store' ) parser.add_argument( "prefix", help="The prefix of the containing folder on disk", action='store' ) parser.add_argument( "--rehash", help="Disregard any existing previously generated" + "hashes, recreate them on this run", action="store_true" ) parser.add_argument( "--scriptloc", help="Specify and alternate script location", action="store" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: # Begin module code # pythonPath = 'python3' if not args.scriptloc: scriptsLoc = dirname(realpath(__file__))+"/" else: scriptsLoc = args.scriptloc appendArgs = [] if args.rehash: appendArgs.append('--rehash') if args.verbosity: appendArgs.append('-b') appendArgs.append(args.verbosity) mvArgs = [pythonPath, scriptsLoc+'ldr_staging_moveFiles.py', args.item, args.root, args.dest_root, args.prefix, "--chain"] mvArgs = mvArgs+appendArgs mvCommand = BashCommand(mvArgs) assert(mvCommand.run_command()[0]) print("\n".join(mvCommand.get_data()[1].stdout.split('\n'))) for line in mvCommand.get_data()[1].stdout.split('\n'): if match('^\[CRITICAL\]', line): print("Critical error detected. Exiting") exit(1) with open('/tmp/folderName.txt', 'r') as f: try: folder = f.read() except Exception as e: logger.critical("ENDS: Failure in reading chain file in tmp. " + "({})".format(e)) origHashArgs = [pythonPath, scriptsLoc+'ldr_staging_originHash.py', args.item, args.root, args.dest_root, folder] origHashArgs = origHashArgs+appendArgs origHashCommand = BashCommand(origHashArgs) assert(origHashCommand.run_command()[0]) print("\n".join(origHashCommand.get_data()[1].stdout.split('\n'))) for line in origHashCommand.get_data()[1].stdout.split('\n'): if match('^\[CRITICAL\]', line): print("Critical error detected. Exiting") exit(1) stageHashArgs = [pythonPath, scriptsLoc+'ldr_staging_stagingHash.py', args.dest_root, folder] stageHashArgs = stageHashArgs+appendArgs stageHashCommand = BashCommand(stageHashArgs) assert(stageHashCommand.run_command()[0]) print("\n".join(stageHashCommand.get_data()[1].stdout.split('\n'))) for line in stageHashCommand.get_data()[1].stdout.split('\n'): if match('^\[CRITICAL\]', line): print("Critical error detected. Exiting") exit(1) auditArgs = [pythonPath, scriptsLoc+'ldr_staging_audit.py', args.dest_root, folder] auditArgs = auditArgs+appendArgs auditCommand = BashCommand(auditArgs) assert(auditCommand.run_command()[0]) print("\n".join(auditCommand.get_data()[1].stdout.split('\n'))) for line in auditCommand.get_data()[1].stdout.split('\n'): if match('^\[CRITICAL\]', line): print("Critical error detected. Exiting") exit(1) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument( "-v", help="See the version of this program", action="version", version=__version__ ) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against " + "a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) parser.add_argument( "dest_root", help="Enter the destination root path", action='store' ) parser.add_argument( "prefix", help="The prefix of the containing folder on disk", action='store' ) parser.add_argument( "--rehash", help="Disregard any existing previously generated " + "hashes, recreate them on this run", action="store_true" ) parser.add_argument( "--chain", help="Write the prefix+num to stdout, for chaining " + "this command into the others via some " + "intermediate connection", action="store_true" ) parser.add_argument( "--weird-root", help="If for some reason you deliberately want to " + "generate a strange item root structure which " + "doesn't reflect rsyncs path interpretation " + "behavior pass this option", default=False, action="store_true" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.item[-1] == "/" and args.item != args.root and not args.weird_root: logger.critical("Root appears to not conform to rsync path specs.") exit(1) # End user specified log instantiation # try: # Begin module code # validation = ValidateBase(args.dest_root) if validation[0] != True: logger.critical("Your staging root isn't valid!") exit(1) else: stageRoot = join(*validation[1:]) destinationAdminRoot = join(stageRoot, 'admin/') destinationDataRoot = join(stageRoot, 'data/') prefix = args.prefix if not prefix[-1].isdigit(): destFolder = prefixToFolder(destinationDataRoot, prefix) logger.info("Creating new data and admin directories for your " + "prefix: "+destFolder) destinationAdminFolder = join(destinationAdminRoot, destFolder) destinationDataFolder = join(destinationDataRoot, destFolder) mkAdminDirArgs = ['mkdir', destinationAdminFolder] mkAdminDirComm = BashCommand(mkAdminDirArgs) assert(mkAdminDirComm.run_command()[0]) logger.debug("mkAdminDir output begins") logger.debug(mkAdminDirComm.get_data()[1].args) logger.debug(mkAdminDirComm.get_data()[1].returncode) logger.debug(mkAdminDirComm.get_data()[1].stdout) logger.debug("mkAdminDir output ends") mkDataDirArgs = ['mkdir', destinationDataFolder] mkDataDirComm = BashCommand(mkDataDirArgs) assert(mkDataDirComm.run_command()[0]) logger.debug("mkDataDir output begins") logger.debug(mkDataDirComm.get_data()[1].args) logger.debug(mkDataDirComm.get_data()[1].returncode) logger.debug(mkDataDirComm.get_data()[1].stdout) logger.debug("mkAdminDir output ends") assert(isdir(destinationAdminFolder)) assert(isdir(destinationDataFolder)) else: destFolder = args.prefix logger.info("Attempting to resume transfer into "+destFolder) destinationAdminFolder = join(destinationAdminRoot, destFolder) destinationDataFolder = join(destinationDataRoot, destFolder) for folder in [destinationAdminFolder, destinationDataFolder]: if not exists(folder): logger.critical('It looks like you are trying to resume ' + 'a transfer, but a corresponding data ' + 'or admin folder is missing! Please ' + 'remedy this and try again! Exiting (1)') exit(1) stagingDebugLog = DebugFileHandler( join(destinationAdminFolder, 'log.txt') ) logger.addHandler(stagingDebugLog) logger.info("Beginning rsync") rsyncArgs = ['rsync', '-avz', args.item, destinationDataFolder] rsyncCommand = BashCommand(rsyncArgs) assert(rsyncCommand.run_command()[0]) with open(join(destinationAdminFolder, 'rsyncFromOrigin.txt'), 'a') as f: f.write(str(rsyncCommand.get_data()[1])+'\n') if rsyncCommand.get_data()[1].returncode != 0: logger.warn("Rsync exited with a non-zero return code: " + str(rsyncCommand.get_data()[1].returncode)) logger.debug("Rsync output begins") logger.debug(rsyncCommand.get_data()[1].args) logger.debug(rsyncCommand.get_data()[1].returncode) for line in rsyncCommand.get_data()[1].stdout.split('\n'): logger.debug(line) logger.debug("Rsync output ends") logger.info("Rsync complete.") if args.chain: try: folderName = split(destinationDataFolder)[1] with open('/tmp/folderName.txt', 'w') as f: f.write(folderName) except Exception as e: logger.critical("ENDS: Failure in writing to tmp for chaining.") # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) parser.add_argument( "--acquisition-record", '-a', help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection", action='append' ) parser.add_argument( "--out-file", '-o', help="The location where the full record should be " + " written to disk.", required=True, action="append" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: # Begin module code # # Keep in mind that population order here matters a lot in terms of # how much input the user will be asked for. # Instantiate a blank record with all our fields set to a blank string, # for bounding loops and no funny business when we try and print it. logger.info("Instantiating Record") record = instantiateRecord() # Map our defaults right into the record. logger.info("Mapping defaults") meldRecord(record, RecordFieldsDefaults(), DummyReader, DummyMapper) # Read all the digital acquisition forms, # populate the record with their info, address conflicts logger.info("Reading and mapping digital acquisition records.") for acqRecord in args.acquisition_record: meldRecord(record, acqRecord, ReadAcquisitionRecord, AcquisitionRecordMapping) # Manual input loop logger.info("Beginning Manual Input Loop") manualInput(record) # Run some automated processing over the record to clean up # certain values if required. logger.info("Beginning attempts at automated boolean interpretation") record = booleanLoop(record, RecordFieldsBooleans()) # Validate the record fields against their stored regexes logger.info("Validating...") record = validate(record, RecordFieldsValidation()) # File level information population logger.info("Generating file info...") record['fileInfo'] = generateFileEntries(args.root, args.item) logger.info("Computing total size") record['totalDigitalSize'] = computeTotalFileSizeFromRecord(record) # Write two records, one which contains the entirety of the record, # including potential internal information, to an internal source, # and another which contains information pertinent to the LDR into # the admin directory logger.info("Writing whole record to out files: " + str(args.out_file)) for filepath in args.out_file: assert(writeNoClobber(record, filepath)) logger.info("Creating subrecord") pubRecord = createSubRecord(record, LDRFields()) logger.info("Attempting to write LDR subrecord into staging structure.") ldrRecordPath = None validation = ValidateBase(args.item) if validation[0] == True: ldrRecordPath = join(*validation[1:], "admin", 'record.json') else: logger.warn("You don't seem to have pointed the script at a " + "fully qualified staging structure. Please manually " + "specify a location to save the LDR record to, " + "otherwise leave this line blank to save only " + "the full record." ) while ldrRecordPath == None: ldrRecordPath = input("LDR Record Path: ") if ldrRecordPath == "": break if len(ldrRecordPath) > 0: ldrRecordPath = abspath(expandvars(ldrRecordPath)) print("Attempted abspath " + ldrRecordPath) if not isdir(ldrRecordPath): ldrRecordPath = None if ldrRecordPath != "": writeNoClobber(pubRecord, ldrRecordPath) logger.info("LDR Record written") else: logger.info("LDR Record generation skipped.") logger.info(json.dumps(record, indent=4, sort_keys=True)) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser( description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by " + __author__ + " " + __email__, ) parser.add_argument("-v", help="See the version of this program", action="version", version=__version__) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( "-b", "--verbosity", help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs="?", const="INFO" ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument("-d", "--debugging", help="set debugging logging", action="store_true") # optionally save the log to a file. # Set a location or use the default constant parser.add_argument("-l", "--log_loc", help="save logging to a file", dest="log_loc") parser.add_argument("root", help="Enter the root of the directory path", action="store") parser.add_argument("ark", help="Enter the ark of placeholder", action="store") parser.add_argument("ead", help="Enter the EADID suffix", action="store") parser.add_argument("accno", help="Enter the accession number", action="store") try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"]: logger.critical( "You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'" ) return 1 if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return 1 # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: # Begin module code # root = args.root ark = args.ark ead = args.ead accno = args.accno logger.debug("User supplied root: {}".format(root)) logger.debug("User supplied ark: {}".format(ark)) logger.debug("User supplied EAD: {}".format(ead)) logger.debug("User supplied accno: {}".format(accno)) if exists(join(root, ark)): logger.critical("ENDS: The specified location already exists.") exit(1) newStructure = createStagingStructure(root, ark, ead, accno) logger.info("New staging root is: " + newStructure) print("New staging root is:\n" + newStructure) # End module code # logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception (" + str(e) + ")") return 1
def main(): # Master log instantiation begins # global masterLog masterLog = MasterLogger() # Master log instantiation ends # # Application specific log instantation begins # global logger logger = masterLog.getChild(__name__) f = UserAndIPFilter() termHandler = DefaultTermHandler() logger.addHandler(termHandler) logger.addFilter(f) logger.info("BEGINS") # Application specific log instantation ends # # Parser instantiation begins # parser = ArgumentParser(description="[A brief description of the utility]", epilog="Copyright University of Chicago; " + "written by "+__author__ + " "+__email__) parser.add_argument( "-v", help="See the version of this program", action="version", version=__version__ ) # let the user decide the verbosity level of logging statements # -b sets it to INFO so warnings, errors and generic informative statements # will be logged parser.add_argument( '-b', '--verbosity', help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs='?', const='INFO' ) # -d is debugging so anything you want to use a debugger gets logged if you # use this level parser.add_argument( '-d', '--debugging', help="set debugging logging", action='store_true' ) # optionally save the log to a file. # Set a location or use the default constant parser.add_argument( '-l', '--log_loc', help="save logging to a file", dest="log_loc", ) parser.add_argument( "item", help="Enter a noid for an accession or a " + "directory path that you need to validate against" + " a type of controlled collection" ) parser.add_argument( "root", help="Enter the root of the directory path", action="store" ) parser.add_argument( "dest_root", help="Enter the destination root path", action='store' ) parser.add_argument( "containing_folder", help="The name of the containing folder on disk " + "(prefix+number)", action='store' ) parser.add_argument( "--rehash", help="Disregard any existing previously generated " + "hashes, recreate them on this run", action="store_true" ) try: args = parser.parse_args() except SystemExit: logger.critical("ENDS: Command line argument parsing failed.") exit(1) # Begin argument post processing, if required # if args.verbosity and args.verbosity not in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: logger.critical("You did not pass a valid argument to the verbosity \ flag! Valid arguments include: \ 'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'") return(1) if args.log_loc: if not exists(split(args.log_loc)[0]): logger.critical("The specified log location does not exist!") return(1) # End argument post processing # # Begin user specified log instantiation, if required # if args.log_loc: fileHandler = DefaultFileHandler(args.log_loc) logger.addHandler(fileHandler) if args.verbosity: logger.removeHandler(termHandler) termHandler = DefaultTermHandlerAtLevel(args.verbosity) logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity) logger.addHandler(fileHandler) if args.debugging: logger.removeHandler(termHandler) termHandler = DebugTermHandler() logger.addHandler(termHandler) if args.log_loc: logger.removeHandler(fileHandler) fileHandler = DebugFileHandler(args.log_loc) logger.addHandler(fileHandler) # End user specified log instantiation # try: if args.item[-1] == "/" and args.item != args.root: logger.warn("It looks like you may have set the root incorrectly.") wrongRootGoAnyways = input("Are you sure you want to continue? " + "(y/n)\n") if wrongRootGoAnyways is not 'y': exit(1) validation = ValidateBase(args.dest_root) if validation[0] == True: stageRoot = join(*validation[1:]) else: logger.critical("Your staging root appears to not be valid!") exit(1) destinationAdminRoot = join(stageRoot, 'admin/') containing_folder = args.containing_folder destinationAdminFolder = join(destinationAdminRoot, containing_folder) stagingDebugLog = DebugFileHandler( join(destinationAdminFolder, 'log.txt') ) logger.addHandler(stagingDebugLog) logger.debug("Creating batch from original files.") originalFiles = Batch(args.root, directory=args.item) logger.info("Hashing original files") if args.rehash: logger.info( "Rehash argumnet passed. Not reading existing hashes." ) existingHashes = None if not args.rehash and exists( join( destinationAdminFolder, 'fixityFromOrigin.txt' )): existingHashes = ReadExistingFixityLog( join( destinationAdminFolder, 'fixityFromOrigin.txt' ) ) WriteFixityLog(join( destinationAdminFolder, 'fixityFromOrigin.txt' ), originalFiles, existingHashes=existingHashes) logger.info("ENDS: COMPLETE") return 0 except KeyboardInterrupt: logger.error("ENDS: Program aborted manually") return 131 except Exception as e: logger.critical("ENDS: Exception ("+str(e)+")") return 1