def run():
basename = os.path.basename(sys.argv[0])
parser = argparse.ArgumentParser(prog=basename,
description='''A debugging utility to view record groups. This
is useful in viewing records without having to do a
complete database ingest.''',
epilog='''example: condorLogInfo.py -c 630.000.000 -f srp_2012_0925_160117/*nodes.log''')
parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", help="verbose")
parser.add_argument("-c", "--condorid", action="store", default=None, dest="condorIds", help="print only condorId(s)", nargs="+", type=str, required=False)
parser.add_argument("-f", "--filenames", action="store", default=None, dest="filenames", help="condor log files", nargs="+", type=str, required=True)
args = parser.parse_args()
for filename in args.filenames:
if not os.path.exists(filename):
if args.verbose:
print "warning: file %s not found " % filename
continue
reader = Reader(filename)
records = reader.getRecords()
# print all the records
if args.condorIds is None:
for job in records:
printRecords(records, job, args.verbose)
else:
# print only the records with these condorIdss
for job in args.condorIds:
if job in records:
printRecords(records, job, args.verbose)
def ingest(self, metrics, filename):
"""Read in a Condor event log, group records per Condor ID,
consolidate that information, and put it into database tables.
@param metrics: a Condor metrics file
@param filename: a Condor event log
"""
# read and parse in the Condor log
reader = Reader(metrics, filename)
# get the record groups, which are grouped by condor id
records = reader.getRecords()
classifier = Classifier()
for job in records:
entries, totalsRecord, updateEntries = \
classifier.classify(records[job])
# add submission records
for ent in entries:
cmd, args = ent.getInsertQuery(self.submissionsTable)
self.dbm.execCommand0(cmd, *args)
# add update records
for ent in updateEntries:
cmd, args = ent.getInsertQuery(self.updatesTable)
self.dbm.execCommand0(cmd, *args)
# add total entry
cmd, args = totalsRecord.getInsertQuery(self.totalsTable)
self.dbm.execCommand0(cmd, *args)
def ingest(self, metrics, filename):
"""Read in a Condor event log, group records per Condor ID,
consolidate that information, and put it into database tables.
@param metrics: a Condor metrics file
@param filename: a Condor event log
"""
# read and parse in the Condor log
reader = Reader(metrics, filename)
# get the record groups, which are grouped by condor id
records = reader.getRecords()
classifier = Classifier()
for job in records:
entries, totalsRecord, updateEntries = \
classifier.classify(records[job])
# add submission records
for ent in entries:
cmd, args = ent.getInsertQuery(self.submissionsTable)
self.dbm.execCommand0(cmd, *args)
# add update records
for ent in updateEntries:
cmd, args = ent.getInsertQuery(self.updatesTable)
self.dbm.execCommand0(cmd, *args)
# add total entry
cmd, args = totalsRecord.getInsertQuery(self.totalsTable)
self.dbm.execCommand0(cmd, *args)
def setUp(self):
filename = os.path.join("tests","testfiles","terminated.log")
reader = Reader(filename)
self.records = reader.getRecords()
# Condor doesn't emit the the current year in records. The classifier
# code assumes it's the current year, and prepends that, so we have
# to assume that here as well when testing for the date.
self.year = str(date.today().year)
def setUp(self):
pkgDir = os.path.abspath(os.path.dirname(__file__))
metrics = os.path.join(pkgDir, "testfiles", "test.metrics")
filename = os.path.join(pkgDir, "testfiles", "terminated.log")
reader = Reader(metrics, filename)
self.records = reader.getRecords()
def setUp(self):
pkgDir = os.path.abspath(os.path.dirname(__file__))
metrics = os.path.join(pkgDir, "testfiles", "test.metrics")
filename = os.path.join(pkgDir, "testfiles", "reader_test.log")
reader = Reader(metrics, filename)
self.records = reader.getRecords()
def setUp(self):
filename = os.path.join("tests","testfiles","reader_test.log")
reader = Reader(filename)
self.records = reader.getRecords()