def txtConverter(item):
if not exists(item.get_file_path()+'.presform.txt'):
fileName, fileExtension = splitext(item.get_file_path())
mkdirArgs = ['mkdir', '-p', '/tmp/officeConv']
mkdirCommand = BashCommand(mkdirArgs)
mkdirCommand.set_timeout(timeout)
mkdirCommand.run_command()
officeConvertArgs = ['/Applications/LibreOffice.app/Contents/MacOS/soffice',
'--headless', '--convert-to', 'txt:Text',
'--outdir', '/tmp/officeConv',
item.get_file_path()]
officeConvertCommand = BashCommand(officeConvertArgs)
officeConvertCommand.set_timeout(timeout)
officeConvertCommand.run_command()
cpCommandArgs = ['cp', '/tmp/officeConv/'+basename(fileName)+'.txt',
item.get_file_path()+'.presform.txt']
cpCommand = BashCommand(cpCommandArgs)
cpCommand.set_timeout(timeout)
cpCommand.run_command()
rmCommandArgs = ['rm', '-r', '/tmp/officeConv']
rmCommand = BashCommand(rmCommandArgs)
rmCommand.set_timeout(timeout)
rmCommand.run_command()
logger.debug(officeConvertCommand.get_data())
return officeConvertCommand.get_data()
else:
logger.info("Office (CSV) preservation format for file exists. " +
"Not Clobbering.")
return(None, None)
def imageConverter(item):
if not exists(item.get_file_path()+'.presform.tif'):
imageConvertArgs=['ffmpeg','-n','-i',item.get_file_path(),item.get_file_path()+'.presform.tif']
imageConvertCommand=BashCommand(imageConvertArgs)
imageConvertCommand.run_command()
imageConvertCommand.read_data()
logger.debug(imageConvertCommand.get_data())
return imageConvertCommand.get_data()
else:
logger.info("Image (tif) preservaiton format for file exists. Not Clobbering.")
def videoConverter(item):
if not exists(item.get_file_path()+'.presform.avi'):
videoConvertArgs=['ffmpeg','-n','-i',item.get_file_path(),'-vcodec','rawvideo','-acodec','pcm_u24le','-pix_fmt','uyvy422','-vtag','2vuy',item.get_file_path()+".presform.avi"]
videoConvertCommand=BashCommand(videoConvertArgs)
videoConvertCommand.run_command()
videoConvertCommand.read_data()
logger.debug(videoConvertCommand.get_data())
return videoConvertCommand.get_data()
else:
logger.info("Video (avi) preservation format for file exists. Not Clobbering.")
return (None,None)
def audioConverter(item):
if not exists(item.get_file_path()+'.presform.wav'):
audioConvertArgs=['ffmpeg','-n','-i',item.get_file_path(),item.get_file_path()+'.presform.wav']
audioConvertCommand=BashCommand(audioConvertArgs)
audioConvertCommand.run_command()
audioConvertCommand.read_data()
logger.debug(audioConvertCommand.get_data())
return audioConvertCommand.get_data()
else:
logger.info("Audio (wav) preservation format for file exists. Not Clobbering.")
return (None,None)
def gifConverter(item):
if not exists(item.get_file_path()+'.presform'):
mkdirArgs=['mkdir',item.get_file_path()+".presform"]
mkdirCommand=BashCommand(mkdirArgs)
mkdirCommand.run_command()
gifConvertArgs=['ffmpeg','-n','-i',item.get_file_path(),item.get_file_path()+'.presform/output%04d.presform.tif']
gifConvertCommand=BashCommand(gifConvertArgs)
gifConvertCommand.run_command()
gifConvertCommand.read_data()
logger.debug(gifConvertCommand.get_data())
return gifConvertCommand.get_data()
else:
logger.info("Image (tif) preservation format for file exists. Not Clobbering.")
def createStagingStructure(root, ark, ead, accno):
from os.path import join, exists
from uchicagoldr.bash_cmd import BashCommand
assert(exists(root))
mkAdminDirArgs = ['mkdir', '-p', join(root, ark, ead, accno, "admin")]
mkAdminDirCommand = BashCommand(mkAdminDirArgs)
assert(mkAdminDirCommand.run_command()[0])
assert(mkAdminDirCommand.get_data()[1].returncode == 0)
mkDataDirArgs = ['mkdir', join(root, ark, ead, accno, "data")]
mkDataDirCommand = BashCommand(mkDataDirArgs)
assert(mkDataDirCommand.run_command()[0])
assert(mkDataDirCommand.get_data()[1].returncode == 0)
return join(root, ark)
def zipConverter(item):
if not exists(item.get_file_path()+'.presform.extracted'):
unzipCommandArgs=['7z','x','-o'+item.get_file_path()+'.presform.extracted',item.get_file_path()]
unzipCommand=BashCommand(unzipCommandArgs)
unzipCommand.run_command()
unzipCommand.read_data()
print(unzipCommand.read_data())
b=Batch(root,item.get_file_path()+'.presform.extracted')
for item in b.find_items(from_directory=True):
itemStack.append(item)
return unzipCommand.get_data()
else:
logger.info("Already extracted.")
def htmlConverter(item):
if not exists(item.get_file_path()+'.presform.pdf'):
originalFilePath = item.get_file_path()
intermediaryFilePath = originalFilePath+'.intermediary.pdf'
wkhtmltopdfArgs = ['wkhtmltopdf', item.get_file_path(),
intermediaryFilePath]
wkhtmltopdfCommand = BashCommand(wkhtmltopdfArgs)
wkhtmltopdfCommand.set_timeout(timeout)
wkhtmltopdfCommand.run_command()
i = Item(intermediaryFilePath, root)
itemStack.append(i)
return wkhtmltopdfCommand.get_data()
else:
return (None, None)
def main():
log_format = Formatter( \
"[%(levelname)s] %(asctime)s " + \
"= %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S" \
)
global logger
logger = getLogger( \
"lib.uchicago.repository.logger" \
)
logger.setLevel('DEBUG')
# start of parser boilerplate
parser = ArgumentParser(description="A command line utility for staging physical media",
epilog="Copyright University of Chicago; " + \
"written by "+__author__ + \
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument( \
'-b','-verbose',help="set verbose logging",
action='store_const',dest='log_level',
const=INFO,default='INFO' \
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument( \
'-d','--debugging',help="set debugging logging",
action='store_const',dest='log_level',
const=DEBUG,default='DEBUG' \
)
# optionally save the log to a file. set a location or use the default constant
parser.add_argument( \
'-l','--log_loc',help="save logging to a file",
dest="log_loc",
\
)
parser.add_argument( \
'--log_verb',help="Set a separate verbosity for the log written to disk, if desired",
dest="log_verb",default=None
\
)
parser.add_argument("item", help="Enter a noid for an accession or a " + \
"directory path that you need to validate against" + \
" a type of controlled collection"
)
parser.add_argument("root",help="Enter the root of the directory path",
action="store"
)
parser.add_argument("dest_root",help="Enter the destination root path",
action='store'
)
parser.add_argument("prefix",help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument("--rehash",help="Disregard any existing previously generated hashes, recreate them on this run",
action="store_true"
)
parser.add_argument("--chain",help="Write the prefix+num to stdout, for chaining this command into the others via some intermediate connection",
action="store_true"
)
parser.add_argument("--weird-root",help="If for some reason you deliberately want to generate a strange item root structure which doesn't reflect rsyncs path interpretation behavior pass this option",default=False,
action="store_true"
)
args = parser.parse_args()
ch = StreamHandler()
ch.setFormatter(log_format)
ch.setLevel(args.log_level)
logger.addHandler(ch)
if args.log_loc:
fh = FileHandler(args.log_loc)
fh.setFormatter(log_format)
logger.addHandler(fh)
if args.item[-1] == "/" and args.item != args.root and not args.weird_root:
logger.critical("Root appears to not conform to rsync path specs.")
exit(1)
try:
assert(isdir(args.dest_root))
shouldBeEAD=getImmediateSubDirs(args.dest_root)
assert(len(shouldBeEAD)==1)
shouldBeAccNo=getImmediateSubDirs(join(args.dest_root,shouldBeEAD[0]))
assert(len(shouldBeAccNo)==1)
stageRoot=join(join(args.dest_root,shouldBeEAD[0]),shouldBeAccNo[0])
destinationAdminRoot=join(stageRoot,'admin/')
destinationDataRoot=join(stageRoot,'data/')
prefix=args.prefix
if not prefix[-1].isdigit():
existingDataSubDirs=[name for name in getImmediateSubDirs(destinationDataRoot) if prefix in name]
if len(existingDataSubDirs) < 1:
nextNum=str(1)
else:
nums=[]
for directory in existingDataSubDirs:
num=directory.strip(prefix)
nums.append(int(num))
nums.sort()
nextNum=str(nums[-1]+1)
logger.info("Creating new data and admin directories for your prefix: "+prefix+nextNum)
destinationAdminFolder=join(destinationAdminRoot,prefix+nextNum)
destinationDataFolder=join(destinationDataRoot,prefix+nextNum)
mkAdminDirArgs=['mkdir',destinationAdminFolder]
mkAdminDirComm=BashCommand(mkAdminDirArgs)
assert(mkAdminDirComm.run_command()[0])
logger.debug("mkAdminDir output begins")
logger.debug(mkAdminDirComm.get_data()[1].args)
logger.debug(mkAdminDirComm.get_data()[1].returncode)
logger.debug(mkAdminDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
mkDataDirArgs=['mkdir',destinationDataFolder]
mkDataDirComm=BashCommand(mkDataDirArgs)
assert(mkDataDirComm.run_command()[0])
logger.debug("mkDataDir output begins")
logger.debug(mkDataDirComm.get_data()[1].args)
logger.debug(mkDataDirComm.get_data()[1].returncode)
logger.debug(mkDataDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
assert(isdir(destinationAdminFolder))
assert(isdir(destinationDataFolder))
else:
logger.info("Attempting to resume transfer into "+join(destinationDataRoot,prefix))
nextNum=""
destinationAdminFolder=join(destinationAdminRoot,prefix)
assert(isdir(destinationAdminFolder))
destinationDataFolder=join(destinationDataRoot,prefix)
assert(isdir(destinationDataFolder))
stagingDebugLog = FileHandler(join(destinationAdminFolder,'log.txt'))
stagingDebugLog.setFormatter(log_format)
stagingDebugLog.setLevel('DEBUG')
logger.addHandler(stagingDebugLog)
logger.info("Beginning rsync")
rsyncArgs=['rsync','-avz',args.item,destinationDataFolder]
rsyncCommand=BashCommand(rsyncArgs)
assert(rsyncCommand.run_command()[0])
with open(join(destinationAdminFolder,'rsyncFromOrigin.txt'),'a') as f:
f.write(str(rsyncCommand.get_data()[1])+'\n')
if rsyncCommand.get_data()[1].returncode != 0:
logger.warn("Rsync exited with a non-zero return code: "+str(rsyncCommand.get_data()[1].returncode))
logger.debug("Rsync output begins")
logger.debug(rsyncCommand.get_data()[1].args)
logger.debug(rsyncCommand.get_data()[1].returncode)
for line in rsyncCommand.get_data()[1].stdout.split('\n'):
logger.debug(line)
logger.debug("Rsync output ends")
logger.info("Rsync complete.")
if args.chain:
logger.info(prefix+nextNum)
return 0
except KeyboardInterrupt:
logger.error("Program aborted manually")
return 131
def main():
# start of parser boilerplate
parser = ArgumentParser(description="This module is meant to take a batch of files (probably an accession in place) and generate the technical metadata for it.",
epilog="Copyright University of Chicago; " + \
"written by "+__author__ + \
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument( \
'-b','-verbose',help="set verbosity for logging to stdout",
action='store_const',dest='log_level',
const=INFO,default='INFO' \
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument( \
'-d','--debugging',help="set debugging logging",
action='store_const',dest='log_level',
const=DEBUG,default='INFO' \
)
# optionally save the log to a file. set a location or use the default constant
parser.add_argument( \
'-l','--log_loc',help="save logging to a file",
dest="log_loc",
\
)
parser.add_argument( \
'-t','--timeout',help="set a timeout in seconds for any single bash command",
dest='timeout',default=3600,type=int \
)
parser.add_argument("item", help="Enter a noid for an accession or a " + \
"directory path that you need to validate against" + \
" a type of controlled collection"
)
parser.add_argument("root",help="Enter the root of the directory path",
action="store"
)
args = parser.parse_args()
log_format = Formatter( \
"[%(levelname)s] %(asctime)s " + \
"= %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S" \
)
global logger
logger = getLogger( \
"lib.uchicago.repository.logger" \
)
logger.setLevel(DEBUG)
ch = StreamHandler()
ch.setFormatter(log_format)
ch.setLevel(args.log_level)
logger.addHandler(ch)
if args.log_loc:
fh = FileHandler(args.log_loc)
fh.setFormatter(log_format)
logger.addHandler(fh)
try:
fitscommand="fits"
md5command="md5"
shacommand="sha256"
b = Batch(abspath(args.root), abspath(args.item))
for item in b.find_items(from_directory=True):
if ".fits.xml" in item.find_file_name() or ".stif.txt" in item.find_file_name():
continue
item.find_technical_metadata()
if item.has_technical_md:
logger.info(item.get_file_path()+" already has technical metadata. Continuing.")
continue
else:
logger.info("Attempting technical metadata generation for: "+item.get_file_path())
fitsArgs=[fitscommand,'-i',item.get_file_path(),'-o',item.get_file_path()+'.fits.xml']
fitsCommand=BashCommand(fitsArgs)
fitsCommand.set_timeout(args.timeout)
try:
logger.info("Attempting FITS generation for: "+item.get_file_path())
result=fitsCommand.run_command()
if isinstance(result[1],Exception):
raise result[1]
assert(exists(item.get_file_path()+'.fits.xml'))
logger.info("FITS generated for: "+item.get_file_path())
except TimeoutExpired:
logger.warn("FITS generation timed out")
logger.info("Attempting STIF generation")
statArgs=['stat',item.get_file_path()]
statCommand=BashCommand(statArgs)
statCommand.set_timeout(args.timeout)
mimeArgs=['file','-i',item.get_file_path()]
mimeCommand=BashCommand(mimeArgs)
mimeCommand.set_timeout(args.timeout)
fileArgs=['file',item.get_file_path()]
fileCommand=BashCommand(fileArgs)
fileCommand.set_timeout(args.timeout)
assert(statCommand.run_command()[0])
assert(mimeCommand.run_command()[0])
assert(fileCommand.run_command()[0])
md5hash=item.find_md5_hash()
shahash=item.find_sha256_hash
with open(item.get_file_path()+'.stif.txt','w') as f:
f.write(statCommand.get_data()[1].stdout.decode(encoding='UTF-8')+ \
mimeCommand.get_data()[1].stdout.decode(encoding='UTF-8')+ \
fileCommand.get_data()[1].stdout.decode(encoding='UTF-8')+ \
"md5: " + item.find_md5_hash() + '\n'+ \
"sha256: " + item.find_sha256_hash() \
)
assert(exists(item.get_file_path()+'.stif.txt'))
logger.info("STIF generated for: "+item.get_file_path())
item.find_technical_metadata()
assert(item.has_technical_md)
logger.info("Technical metadata generation complete for: "+item.get_file_path())
return 0
except KeyboardInterrupt:
logger.error("Program aborted manually")
return 131
def main():
# start of parser boilerplate
parser = ArgumentParser(description="A command line utility for staging physical media",
epilog="Copyright University of Chicago; " + \
"written by "+__author__ + \
" "+__email__)
parser.add_argument("-v", help="See the version of this program",
action="version", version=__version__)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument( \
'-b','-verbose',help="set verbose logging",
action='store_const',dest='log_level',
const=INFO,default='INFO' \
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument( \
'-d','--debugging',help="set debugging logging",
action='store_const',dest='log_level',
const=DEBUG,default='DEBUG' \
)
# optionally save the log to a file. set a location or use the default constant
parser.add_argument( \
'-l','--log_loc',help="save logging to a file",
dest="log_loc",
\
)
parser.add_argument( \
'--log_verb',help="Set a separate verbosity for the log written to disk, if desired",
dest="log_verb",default=None
\
)
parser.add_argument("item", help="Enter a noid for an accession or a " + \
"directory path that you need to validate against" + \
" a type of controlled collection"
)
parser.add_argument("root",help="Enter the root of the directory path",
action="store"
)
parser.add_argument("dest_root",help="Enter the destination root path",
action='store'
)
parser.add_argument("prefix",help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument("--rehash",help="Disregard any existing previously generated hashes, recreate them on this run",
action="store_true"
)
args = parser.parse_args()
log_format = Formatter( \
"[%(levelname)s] %(asctime)s " + \
"= %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S" \
)
global logger
logger = getLogger( \
"lib.uchicago.repository.logger" \
)
ch = StreamHandler()
ch.setFormatter(log_format)
ch.setLevel(args.log_level)
logger.setLevel('DEBUG')
if args.log_loc:
fh = FileHandler(args.log_loc)
fh.setFormatter(log_format)
logger.addHandler(fh)
logger.addHandler(ch)
try:
pythonPath='python3'
scriptsLoc='/Users/balsamo/repos/uchicago-ldr/bin/'
mvArgs=[pythonPath,scriptsLoc+'ldr_staging_moveFiles.py',args.item,args.root,args.dest_root,args.prefix,"--chain"]
mvCommand=BashCommand(mvArgs)
assert(mvCommand.run_command()[0])
print("\n".join(mvCommand.get_data()[1].stdout.split('\n')))
for line in mvCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]',line):
print("Critical error detected. Exiting")
exit(1)
folder=mvCommand.get_data()[1].stdout.split('=')[-1].rstrip('\n').strip()
origHashArgs=[pythonPath,scriptsLoc+'ldr_staging_originHash.py',args.item,args.root,args.dest_root,folder]
if args.rehash:
origHashArgs.append("--rehash")
origHashCommand=BashCommand(origHashArgs)
assert(origHashCommand.run_command()[0])
print("\n".join(origHashCommand.get_data()[1].stdout.split('\n')))
for line in origHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]',line):
print("Critical error detected. Exiting")
exit(1)
stageHashArgs=[pythonPath,scriptsLoc+'ldr_staging_stagingHash.py',args.dest_root,folder]
if args.rehash:
stageHashArgs.append("--rehash")
stageHashCommand=BashCommand(stageHashArgs)
assert(stageHashCommand.run_command()[0])
print("\n".join(stageHashCommand.get_data()[1].stdout.split('\n')))
for line in stageHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]',line):
print("Critical error detected. Exiting")
exit(1)
auditArgs=[pythonPath,scriptsLoc+'ldr_staging_audit.py',args.dest_root,folder]
auditCommand=BashCommand(auditArgs)
assert(auditCommand.run_command()[0])
print("\n".join(auditCommand.get_data()[1].stdout.split('\n')))
for line in auditCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]',line):
print("Critical error detected. Exiting")
exit(1)
return 0
except KeyboardInterrupt:
logger.error("Program aborted manually")
return 131
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against" +
" a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"prefix",
help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated" +
"hashes, recreate them on this run",
action="store_true"
)
parser.add_argument(
"--scriptloc",
help="Specify and alternate script location",
action="store"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
# End user specified log instantiation #
try:
# Begin module code #
pythonPath = 'python3'
if not args.scriptloc:
scriptsLoc = dirname(realpath(__file__))+"/"
else:
scriptsLoc = args.scriptloc
appendArgs = []
if args.rehash:
appendArgs.append('--rehash')
if args.verbosity:
appendArgs.append('-b')
appendArgs.append(args.verbosity)
mvArgs = [pythonPath, scriptsLoc+'ldr_staging_moveFiles.py',
args.item, args.root, args.dest_root, args.prefix, "--chain"]
mvArgs = mvArgs+appendArgs
mvCommand = BashCommand(mvArgs)
assert(mvCommand.run_command()[0])
print("\n".join(mvCommand.get_data()[1].stdout.split('\n')))
for line in mvCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
with open('/tmp/folderName.txt', 'r') as f:
try:
folder = f.read()
except Exception as e:
logger.critical("ENDS: Failure in reading chain file in tmp. " +
"({})".format(e))
origHashArgs = [pythonPath, scriptsLoc+'ldr_staging_originHash.py',
args.item, args.root, args.dest_root, folder]
origHashArgs = origHashArgs+appendArgs
origHashCommand = BashCommand(origHashArgs)
assert(origHashCommand.run_command()[0])
print("\n".join(origHashCommand.get_data()[1].stdout.split('\n')))
for line in origHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
stageHashArgs = [pythonPath, scriptsLoc+'ldr_staging_stagingHash.py',
args.dest_root, folder]
stageHashArgs = stageHashArgs+appendArgs
stageHashCommand = BashCommand(stageHashArgs)
assert(stageHashCommand.run_command()[0])
print("\n".join(stageHashCommand.get_data()[1].stdout.split('\n')))
for line in stageHashCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
auditArgs = [pythonPath, scriptsLoc+'ldr_staging_audit.py',
args.dest_root, folder]
auditArgs = auditArgs+appendArgs
auditCommand = BashCommand(auditArgs)
assert(auditCommand.run_command()[0])
print("\n".join(auditCommand.get_data()[1].stdout.split('\n')))
for line in auditCommand.get_data()[1].stdout.split('\n'):
if match('^\[CRITICAL\]', line):
print("Critical error detected. Exiting")
exit(1)
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
def main():
# Master log instantiation begins #
global masterLog
masterLog = MasterLogger()
# Master log instantiation ends #
# Application specific log instantation begins #
global logger
logger = masterLog.getChild(__name__)
f = UserAndIPFilter()
termHandler = DefaultTermHandler()
logger.addHandler(termHandler)
logger.addFilter(f)
logger.info("BEGINS")
# Application specific log instantation ends #
# Parser instantiation begins #
parser = ArgumentParser(description="[A brief description of the utility]",
epilog="Copyright University of Chicago; " +
"written by "+__author__ +
" "+__email__)
parser.add_argument(
"-v",
help="See the version of this program",
action="version",
version=__version__
)
# let the user decide the verbosity level of logging statements
# -b sets it to INFO so warnings, errors and generic informative statements
# will be logged
parser.add_argument(
'-b', '--verbosity',
help="set logging verbosity " +
"(DEBUG,INFO,WARN,ERROR,CRITICAL)",
nargs='?',
const='INFO'
)
# -d is debugging so anything you want to use a debugger gets logged if you
# use this level
parser.add_argument(
'-d', '--debugging',
help="set debugging logging",
action='store_true'
)
# optionally save the log to a file.
# Set a location or use the default constant
parser.add_argument(
'-l', '--log_loc',
help="save logging to a file",
dest="log_loc",
)
parser.add_argument(
"item",
help="Enter a noid for an accession or a " +
"directory path that you need to validate against " +
"a type of controlled collection"
)
parser.add_argument(
"root",
help="Enter the root of the directory path",
action="store"
)
parser.add_argument(
"dest_root",
help="Enter the destination root path",
action='store'
)
parser.add_argument(
"prefix",
help="The prefix of the containing folder on disk",
action='store'
)
parser.add_argument(
"--rehash",
help="Disregard any existing previously generated " +
"hashes, recreate them on this run",
action="store_true"
)
parser.add_argument(
"--chain",
help="Write the prefix+num to stdout, for chaining " +
"this command into the others via some " +
"intermediate connection",
action="store_true"
)
parser.add_argument(
"--weird-root",
help="If for some reason you deliberately want to " +
"generate a strange item root structure which " +
"doesn't reflect rsyncs path interpretation " +
"behavior pass this option",
default=False,
action="store_true"
)
try:
args = parser.parse_args()
except SystemExit:
logger.critical("ENDS: Command line argument parsing failed.")
exit(1)
# Begin argument post processing, if required #
if args.verbosity and args.verbosity not in ['DEBUG', 'INFO',
'WARN', 'ERROR', 'CRITICAL']:
logger.critical("You did not pass a valid argument to the verbosity \
flag! Valid arguments include: \
'DEBUG','INFO','WARN','ERROR', and 'CRITICAL'")
return(1)
if args.log_loc:
if not exists(split(args.log_loc)[0]):
logger.critical("The specified log location does not exist!")
return(1)
# End argument post processing #
# Begin user specified log instantiation, if required #
if args.log_loc:
fileHandler = DefaultFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.verbosity:
logger.removeHandler(termHandler)
termHandler = DefaultTermHandlerAtLevel(args.verbosity)
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DefaultFileHandlerAtLevel(args.log_loc,
args.verbosity)
logger.addHandler(fileHandler)
if args.debugging:
logger.removeHandler(termHandler)
termHandler = DebugTermHandler()
logger.addHandler(termHandler)
if args.log_loc:
logger.removeHandler(fileHandler)
fileHandler = DebugFileHandler(args.log_loc)
logger.addHandler(fileHandler)
if args.item[-1] == "/" and args.item != args.root and not args.weird_root:
logger.critical("Root appears to not conform to rsync path specs.")
exit(1)
# End user specified log instantiation #
try:
# Begin module code #
validation = ValidateBase(args.dest_root)
if validation[0] != True:
logger.critical("Your staging root isn't valid!")
exit(1)
else:
stageRoot = join(*validation[1:])
destinationAdminRoot = join(stageRoot, 'admin/')
destinationDataRoot = join(stageRoot, 'data/')
prefix = args.prefix
if not prefix[-1].isdigit():
destFolder = prefixToFolder(destinationDataRoot, prefix)
logger.info("Creating new data and admin directories for your " +
"prefix: "+destFolder)
destinationAdminFolder = join(destinationAdminRoot, destFolder)
destinationDataFolder = join(destinationDataRoot, destFolder)
mkAdminDirArgs = ['mkdir', destinationAdminFolder]
mkAdminDirComm = BashCommand(mkAdminDirArgs)
assert(mkAdminDirComm.run_command()[0])
logger.debug("mkAdminDir output begins")
logger.debug(mkAdminDirComm.get_data()[1].args)
logger.debug(mkAdminDirComm.get_data()[1].returncode)
logger.debug(mkAdminDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
mkDataDirArgs = ['mkdir', destinationDataFolder]
mkDataDirComm = BashCommand(mkDataDirArgs)
assert(mkDataDirComm.run_command()[0])
logger.debug("mkDataDir output begins")
logger.debug(mkDataDirComm.get_data()[1].args)
logger.debug(mkDataDirComm.get_data()[1].returncode)
logger.debug(mkDataDirComm.get_data()[1].stdout)
logger.debug("mkAdminDir output ends")
assert(isdir(destinationAdminFolder))
assert(isdir(destinationDataFolder))
else:
destFolder = args.prefix
logger.info("Attempting to resume transfer into "+destFolder)
destinationAdminFolder = join(destinationAdminRoot, destFolder)
destinationDataFolder = join(destinationDataRoot, destFolder)
for folder in [destinationAdminFolder, destinationDataFolder]:
if not exists(folder):
logger.critical('It looks like you are trying to resume ' +
'a transfer, but a corresponding data ' +
'or admin folder is missing! Please ' +
'remedy this and try again! Exiting (1)')
exit(1)
stagingDebugLog = DebugFileHandler(
join(destinationAdminFolder, 'log.txt')
)
logger.addHandler(stagingDebugLog)
logger.info("Beginning rsync")
rsyncArgs = ['rsync', '-avz', args.item, destinationDataFolder]
rsyncCommand = BashCommand(rsyncArgs)
assert(rsyncCommand.run_command()[0])
with open(join(destinationAdminFolder,
'rsyncFromOrigin.txt'), 'a') as f:
f.write(str(rsyncCommand.get_data()[1])+'\n')
if rsyncCommand.get_data()[1].returncode != 0:
logger.warn("Rsync exited with a non-zero return code: " +
str(rsyncCommand.get_data()[1].returncode))
logger.debug("Rsync output begins")
logger.debug(rsyncCommand.get_data()[1].args)
logger.debug(rsyncCommand.get_data()[1].returncode)
for line in rsyncCommand.get_data()[1].stdout.split('\n'):
logger.debug(line)
logger.debug("Rsync output ends")
logger.info("Rsync complete.")
if args.chain:
try:
folderName = split(destinationDataFolder)[1]
with open('/tmp/folderName.txt', 'w') as f:
f.write(folderName)
except Exception as e:
logger.critical("ENDS: Failure in writing to tmp for chaining.")
# End module code #
logger.info("ENDS: COMPLETE")
return 0
except KeyboardInterrupt:
logger.error("ENDS: Program aborted manually")
return 131
except Exception as e:
logger.critical("ENDS: Exception ("+str(e)+")")
return 1
