def ingest(self, metrics, filename):
"""Read in a Condor event log, group records per Condor ID,
consolidate that information, and put it into database tables.
@param metrics: a Condor metrics file
@param filename: a Condor event log
"""
# read and parse in the Condor log
reader = Reader(metrics, filename)
# get the record groups, which are grouped by condor id
records = reader.getRecords()
classifier = Classifier()
for job in records:
entries, totalsRecord, updateEntries = \
classifier.classify(records[job])
# add submission records
for ent in entries:
cmd, args = ent.getInsertQuery(self.submissionsTable)
self.dbm.execCommand0(cmd, *args)
# add update records
for ent in updateEntries:
cmd, args = ent.getInsertQuery(self.updatesTable)
self.dbm.execCommand0(cmd, *args)
# add total entry
cmd, args = totalsRecord.getInsertQuery(self.totalsTable)
self.dbm.execCommand0(cmd, *args)
def setUp(self):
pkgDir = os.path.abspath(os.path.dirname(__file__))
metrics = os.path.join(pkgDir, "testfiles", "test.metrics")
filename = os.path.join(pkgDir, "testfiles", "terminated.log")
reader = Reader(metrics, filename)
self.records = reader.getRecords()