def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(
description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " + "written by " + __author__ + " " + __email__,
)
parser.add_argument("-v", help="See the version of this program", action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
"-b", "--verbosity", help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs="?", const="INFO"
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument("-d", "--debugging", help="set debugging logging", action="store_true")
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument("-l", "--log_loc", help="save logging to a file", dest="log_loc")
parser.add_argument("dest_root", help="Enter the destination root path", action="store")
parser.add_argument(
"containing_folder", help="The name of the containing folder on disk " + "(prefix+number)", action="store"
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated " + "hashes, recreate them on this run",
action="store_true",
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"]:
logger.critical(
"You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'"
)
return 1
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return 1
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
logger.info("BEGINS")
# Begin module code #
validation = ValidateBase(args.dest_root)
if validation[0] == True:
stageRoot = join(*validation[1:])
else:
logger.critical("ENDS: Your staging root appears to not be valid!")
exit(1)
destinationAdminRoot = join(stageRoot, "admin/")
destinationDataRoot = join(stageRoot, "data/")
containing_folder = args.containing_folder
destinationAdminFolder = join(destinationAdminRoot, containing_folder)
destinationDataFolder = join(destinationDataRoot, containing_folder)
stagingDebugLog = DebugFileHandler(join(destinationAdminFolder, "log.txt"))
logger.addHandler(stagingDebugLog)
logger.debug("Creating batch from moved files.")
movedFiles = Batch(destinationDataFolder, directory=destinationDataFolder)
logger.info("Hashing copied files.")
existingHashes = None
if args.rehash:
logger.info("Rehash argument passed. Not reading existing hashes.")
if not args.rehash and exists(join(destinationAdminFolder, "fixityOnDisk.txt")):
existingHashes = ReadExistingFixityLog(join(destinationAdminFolder, "fixityOnDisk.txt"))
WriteFixityLog(join(destinationAdminFolder, "fixityOnDisk.txt"), movedFiles, existingHashes=existingHashes)
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception (" + str(e) + ")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"containing_folder",
help="The name of the containing folder on disk " +
"(prefix+number)",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated " +
" hashes, recreate them on this run",
action="store_true"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
logger.info("BEGINS")
# Begin module code #
validation = ValidateBase(args.dest_root)
if validation[0] != True:
logger.critical("Your staging root isn't valid!")
exit(1)
else:
stageRoot = join(*validation[1:])
destinationAdminRoot = join(stageRoot, 'admin/')
containing_folder = args.containing_folder
destinationAdminFolder = join(destinationAdminRoot, containing_folder)
stagingDebugLog = DebugFileHandler(
join(destinationAdminFolder, 'log.txt')
)
logger.addHandler(stagingDebugLog)
existingOriginalFileHashes = ReadExistingFixityLog(
join(
destinationAdminFolder,
'fixityFromOrigin.txt'
)
)
existingMovedFileHashes = ReadExistingFixityLog(
join(
destinationAdminFolder,
'fixityOnDisk.txt'
)
)
notMoved = [key for key in existingOriginalFileHashes
if key not in existingMovedFileHashes]
foreignFiles = [key for key in existingMovedFileHashes
if key not in existingOriginalFileHashes]
badHash = [key for key in existingOriginalFileHashes
if key not in notMoved and
existingOriginalFileHashes[key] != existingMovedFileHashes[key]]
for entry in existingOriginalFileHashes:
if entry not in notMoved and entry not in badHash:
logger.debug("GOOD: "+entry+":" +
str(existingOriginalFileHashes[entry]))
elif entry in notMoved:
logger.debug("NOT MOVED: "+entry+":" +
str(existingOriginalFileHashes[entry]))
elif entry in badHash:
logger.debug("BAD HASH: "+entry+":" +
str(existingOriginalFileHashes[entry]))
for entry in foreignFiles:
logger.debug("FOREIGN FILE: "+entry)
logger.info(str(len(existingMovedFileHashes)) +
" file(s) total in the staging area.")
logger.info(str(len(notMoved)) +
" file(s) not copied.")
logger.info(str(len(badHash)) +
" file(s) have a different hash from the origin.")
logger.info(str(len(foreignFiles)) +
" file(s) appear to not have come from the origin.")
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
# Begin module code #
b = Batch(args.root, args.item)
for item in b.find_items(from_directory=True):
print(item.get_file_path())
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
logger.info("BEGINS")
# Begin module code #
logger.info("Validating Base Structure.")
validation = ValidateBase(args.item)
if validation[0] != True:
logger.critical("Your staging base has not validated!")
logger.critical(validation)
exit(1)
dataPath=join(*validation[1:], 'data')
adminPath=join(*validation[1:], 'admin')
logger.info("Checking data directory.")
dataValid = ValidateOrganization(dataPath)
if dataValid[0] != True:
logger.critical("Your data directory is not well formed!")
logger.critical(dataValid)
exit(1)
for x in dataValid[1]['notDirs']:
logger.warn("The following appears in the data dir " +
"but is not a directory: "+x)
logger.info("Checking admin directory.")
adminValid = ValidateOrganization(adminPath,
reqTopFiles=['record.json',
'fileConversions.txt'],
reqDirContents=[
'fixityFromOrigin.txt',
'fixityOnDisk.txt',
'log.txt',
'rsyncFromOrigin.txt']
)
if adminValid[0] != True:
logger.critical("ENDS: Your admin directory is not well formed!")
logger.warn(adminValid)
exit(1)
if dataValid[1]['dirs'] != adminValid[1]['dirs']:
for x in dataValid[1]['dirs']:
if x not in adminValid[1]['dirs']:
logger.warn("Directory appears in data but not admin: "+x)
for x in adminValid[1]['dirs']:
if x not in dataValid[1]['dirs']:
logger.warn("Directory appears in admin but not data: "+x)
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"prefix",
help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated" +
"hashes, recreate them on this run",
action="store_true"
)
parser.add_argument(
"--scriptloc",
help="Specify and alternate script location",
action="store"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
# Begin module code #
pythonPath = 'python3'
if not args.scriptloc:
scriptsLoc = dirname(realpath(__file__))+"/"
else:
scriptsLoc = args.scriptloc
appendArgs = []
if args.rehash:
appendArgs.append('--rehash')
if args.verbosity:
appendArgs.append('-b')
appendArgs.append(args.verbosity)
mvArgs = [pythonPath, scriptsLoc+'ldr_staging_moveFiles.py',
args.item, args.root, args.dest_root, args.prefix, "--chain"]
mvArgs = mvArgs+appendArgs
mvCommand = BashCommand(mvArgs)
assert(mvCommand.run_command()[0])
print("\n".join(mvCommand.get_data()[1].stdout.split('\n')))
for line in mvCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
with open('/tmp/folderName.txt', 'r') as f:
try:
folder = f.read()
except Exception as e:
logger.critical("ENDS: Failure in reading chain file in tmp. " +
"({})".format(e))
origHashArgs = [pythonPath, scriptsLoc+'ldr_staging_originHash.py',
args.item, args.root, args.dest_root, folder]
origHashArgs = origHashArgs+appendArgs
origHashCommand = BashCommand(origHashArgs)
assert(origHashCommand.run_command()[0])
print("\n".join(origHashCommand.get_data()[1].stdout.split('\n')))
for line in origHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
stageHashArgs = [pythonPath, scriptsLoc+'ldr_staging_stagingHash.py',
args.dest_root, folder]
stageHashArgs = stageHashArgs+appendArgs
stageHashCommand = BashCommand(stageHashArgs)
assert(stageHashCommand.run_command()[0])
print("\n".join(stageHashCommand.get_data()[1].stdout.split('\n')))
for line in stageHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
auditArgs = [pythonPath, scriptsLoc+'ldr_staging_audit.py',
args.dest_root, folder]
auditArgs = auditArgs+appendArgs
auditCommand = BashCommand(auditArgs)
assert(auditCommand.run_command()[0])
print("\n".join(auditCommand.get_data()[1].stdout.split('\n')))
for line in auditCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against " +
"a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"prefix",
help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated " +
"hashes, recreate them on this run",
action="store_true"
)
parser.add_argument(
"--chain",
help="Write the prefix+num to stdout, for chaining " +
"this command into the others via some " +
"intermediate connection",
action="store_true"
)
parser.add_argument(
"--weird-root",
help="If for some reason you deliberately want to " +
"generate a strange item root structure which " +
"doesn't reflect rsyncs path interpretation " +
"behavior pass this option",
default=False,
action="store_true"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.item[-1] == "/" and args.item != args.root and not args.weird_root:
logger.critical("Root appears to not conform to rsync path specs.")
exit(1)
# End user specified log instantiation #
try:
# Begin module code #
validation = ValidateBase(args.dest_root)
if validation[0] != True:
logger.critical("Your staging root isn't valid!")
exit(1)
else:
stageRoot = join(*validation[1:])
destinationAdminRoot = join(stageRoot, 'admin/')
destinationDataRoot = join(stageRoot, 'data/')
prefix = args.prefix
if not prefix[-1].isdigit():
destFolder = prefixToFolder(destinationDataRoot, prefix)
logger.info("Creating new data and admin directories for your " +
"prefix: "+destFolder)
destinationAdminFolder = join(destinationAdminRoot, destFolder)
destinationDataFolder = join(destinationDataRoot, destFolder)
mkAdminDirArgs = ['mkdir', destinationAdminFolder]
mkAdminDirComm = BashCommand(mkAdminDirArgs)
assert(mkAdminDirComm.run_command()[0])
logger.debug("mkAdminDir output begins")
logger.debug(mkAdminDirComm.get_data()[1].args)
logger.debug(mkAdminDirComm.get_data()[1].returncode)
logger.debug(mkAdminDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
mkDataDirArgs = ['mkdir', destinationDataFolder]
mkDataDirComm = BashCommand(mkDataDirArgs)
assert(mkDataDirComm.run_command()[0])
logger.debug("mkDataDir output begins")
logger.debug(mkDataDirComm.get_data()[1].args)
logger.debug(mkDataDirComm.get_data()[1].returncode)
logger.debug(mkDataDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
assert(isdir(destinationAdminFolder))
assert(isdir(destinationDataFolder))
else:
destFolder = args.prefix
logger.info("Attempting to resume transfer into "+destFolder)
destinationAdminFolder = join(destinationAdminRoot, destFolder)
destinationDataFolder = join(destinationDataRoot, destFolder)
for folder in [destinationAdminFolder, destinationDataFolder]:
if not exists(folder):
logger.critical('It looks like you are trying to resume ' +
'a transfer, but a corresponding data ' +
'or admin folder is missing! Please ' +
'remedy this and try again! Exiting (1)')
exit(1)
stagingDebugLog = DebugFileHandler(
join(destinationAdminFolder, 'log.txt')
)
logger.addHandler(stagingDebugLog)
logger.info("Beginning rsync")
rsyncArgs = ['rsync', '-avz', args.item, destinationDataFolder]
rsyncCommand = BashCommand(rsyncArgs)
assert(rsyncCommand.run_command()[0])
with open(join(destinationAdminFolder,
'rsyncFromOrigin.txt'), 'a') as f:
f.write(str(rsyncCommand.get_data()[1])+'\n')
if rsyncCommand.get_data()[1].returncode != 0:
logger.warn("Rsync exited with a non-zero return code: " +
str(rsyncCommand.get_data()[1].returncode))
logger.debug("Rsync output begins")
logger.debug(rsyncCommand.get_data()[1].args)
logger.debug(rsyncCommand.get_data()[1].returncode)
for line in rsyncCommand.get_data()[1].stdout.split('\n'):
logger.debug(line)
logger.debug("Rsync output ends")
logger.info("Rsync complete.")
if args.chain:
try:
folderName = split(destinationDataFolder)[1]
with open('/tmp/folderName.txt', 'w') as f:
f.write(folderName)
except Exception as e:
logger.critical("ENDS: Failure in writing to tmp for chaining.")
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"--acquisition-record", '-a',
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection",
action='append'
)
parser.add_argument(
"--out-file", '-o',
help="The location where the full record should be " +
" written to disk.",
required=True,
action="append"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
# Begin module code #
# Keep in mind that population order here matters a lot in terms of
# how much input the user will be asked for.
# Instantiate a blank record with all our fields set to a blank string,
# for bounding loops and no funny business when we try and print it.
logger.info("Instantiating Record")
record = instantiateRecord()
# Map our defaults right into the record.
logger.info("Mapping defaults")
meldRecord(record, RecordFieldsDefaults(), DummyReader, DummyMapper)
# Read all the digital acquisition forms,
# populate the record with their info, address conflicts
logger.info("Reading and mapping digital acquisition records.")
for acqRecord in args.acquisition_record:
meldRecord(record, acqRecord, ReadAcquisitionRecord,
AcquisitionRecordMapping)
# Manual input loop
logger.info("Beginning Manual Input Loop")
manualInput(record)
# Run some automated processing over the record to clean up
# certain values if required.
logger.info("Beginning attempts at automated boolean interpretation")
record = booleanLoop(record, RecordFieldsBooleans())
# Validate the record fields against their stored regexes
logger.info("Validating...")
record = validate(record, RecordFieldsValidation())
# File level information population
logger.info("Generating file info...")
record['fileInfo'] = generateFileEntries(args.root, args.item)
logger.info("Computing total size")
record['totalDigitalSize'] = computeTotalFileSizeFromRecord(record)
# Write two records, one which contains the entirety of the record,
# including potential internal information, to an internal source,
# and another which contains information pertinent to the LDR into
# the admin directory
logger.info("Writing whole record to out files: " +
str(args.out_file))
for filepath in args.out_file:
assert(writeNoClobber(record, filepath))
logger.info("Creating subrecord")
pubRecord = createSubRecord(record, LDRFields())
logger.info("Attempting to write LDR subrecord into staging structure.")
ldrRecordPath = None
validation = ValidateBase(args.item)
if validation[0] == True:
ldrRecordPath = join(*validation[1:], "admin", 'record.json')
else:
logger.warn("You don't seem to have pointed the script at a " +
"fully qualified staging structure. Please manually " +
"specify a location to save the LDR record to, " +
"otherwise leave this line blank to save only " +
"the full record."
)
while ldrRecordPath == None:
ldrRecordPath = input("LDR Record Path: ")
if ldrRecordPath == "":
break
if len(ldrRecordPath) > 0:
ldrRecordPath = abspath(expandvars(ldrRecordPath))
print("Attempted abspath " + ldrRecordPath)
if not isdir(ldrRecordPath):
ldrRecordPath = None
if ldrRecordPath != "":
writeNoClobber(pubRecord, ldrRecordPath)
logger.info("LDR Record written")
else:
logger.info("LDR Record generation skipped.")
logger.info(json.dumps(record, indent=4, sort_keys=True))
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(
description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " + "written by " + __author__ + " " + __email__,
)
parser.add_argument("-v", help="See the version of this program", action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
"-b", "--verbosity", help="set logging verbosity " + "(DEBUG,INFO,WARN,ERROR,CRITICAL)", nargs="?", const="INFO"
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument("-d", "--debugging", help="set debugging logging", action="store_true")
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument("-l", "--log_loc", help="save logging to a file", dest="log_loc")
parser.add_argument("root", help="Enter the root of the directory path", action="store")
parser.add_argument("ark", help="Enter the ark of placeholder", action="store")
parser.add_argument("ead", help="Enter the EADID suffix", action="store")
parser.add_argument("accno", help="Enter the accession number", action="store")
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"]:
logger.critical(
"You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'"
)
return 1
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return 1
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc, args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
# Begin module code #
root = args.root
ark = args.ark
ead = args.ead
accno = args.accno
logger.debug("User supplied root: {}".format(root))
logger.debug("User supplied ark: {}".format(ark))
logger.debug("User supplied EAD: {}".format(ead))
logger.debug("User supplied accno: {}".format(accno))
if exists(join(root, ark)):
logger.critical("ENDS: The specified location already exists.")
exit(1)
newStructure = createStagingStructure(root, ark, ead, accno)
logger.info("New staging root is: " + newStructure)
print("New staging root is:\n" + newStructure)
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception (" + str(e) + ")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"containing_folder",
help="The name of the containing folder on disk " +
"(prefix+number)",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated " +
"hashes, recreate them on this run",
action="store_true"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
if args.item[-1] == "/" and args.item != args.root:
logger.warn("It looks like you may have set the root incorrectly.")
wrongRootGoAnyways = input("Are you sure you want to continue? " +
"(y/n)\n")
if wrongRootGoAnyways is not 'y':
exit(1)
validation = ValidateBase(args.dest_root)
if validation[0] == True:
stageRoot = join(*validation[1:])
else:
logger.critical("Your staging root appears to not be valid!")
exit(1)
destinationAdminRoot = join(stageRoot, 'admin/')
containing_folder = args.containing_folder
destinationAdminFolder = join(destinationAdminRoot, containing_folder)
stagingDebugLog = DebugFileHandler(
join(destinationAdminFolder, 'log.txt')
)
logger.addHandler(stagingDebugLog)
logger.debug("Creating batch from original files.")
originalFiles = Batch(args.root, directory=args.item)
logger.info("Hashing original files")
if args.rehash:
logger.info(
"Rehash argumnet passed. Not reading existing hashes."
)
existingHashes = None
if not args.rehash and exists(
join(
destinationAdminFolder, 'fixityFromOrigin.txt'
)):
existingHashes = ReadExistingFixityLog(
join(
destinationAdminFolder, 'fixityFromOrigin.txt'
)
)
WriteFixityLog(join(
destinationAdminFolder,
'fixityFromOrigin.txt'
),
originalFiles, existingHashes=existingHashes)
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
