def parse_accounting_line(line, lineno):
""" Parse one line from accounting log and insert the data into DB.
"""
global server,last_event_time
logger = logging.getLogger("goove_updater")
cursor = connection.cursor()
try:
date,event,fulljobid,attrs = line.split(';')
except ValueError:
logger.warning("skipping invalid line %d: '%s'" % (lineno,line))
return
logger.debug("Processing accounting line: %s:%s:%s ..." %(date, event, fulljobid))
last_event_time = datetime.datetime.strptime(date, "%m/%d/%Y %H:%M:%S")
if last_event_time < server.lastacc_update:
logger.info("Ignoring accounting event with datetime %s" % str(last_event_time))
return
# We ignore PBSPro Licensing lines (it is not job related)
if event=='L':
logger.debug("Ignoring licensing line")
return
attrdir = {}
try:
for key,val in map(lambda x: x.split('=',1), attrs.split()):
attrdir[key] = val
except ValueError:
logger.warning("Skipping line with invalid attribues %d: '%s'" % (lineno,attrs))
jobid_name, server_name = JOBID_REGEX.search(fulljobid).groups()
if server_name!=server.name:
logger.warning("The name of the server in jobid: %s differs from server name in database: %s" % (server_name,server.name))
d,t = date.split(' ')
m,d,y = d.split('/')
timestamp='%s-%s-%s %s' % (y,m,d,t)
retval = cursor.execute("INSERT IGNORE INTO trqacc_accountingevent (timestamp, type, short_jobid, server_id) VALUES (%s,%s,%s,%s)", [timestamp, event, jobid_name, server.id])
if retval==1:
# FIXME: I am not sure this is thread safe - should call something like C-API mysql_insert_id();
cursor.execute("SELECT LAST_INSERT_ID()")
row = cursor.fetchone()
ae_id = row[0]
for key,val in attrdir.items():
ea,created = getEventAttribute(key)
logger.debug("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)" % (ae_id, ea.id, val))
cursor.execute("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)", [ae_id, ea.id, val])
else:
logger.warning("Tried to insert already present accounting event - skipping, check your logs for: %s:%s:%s ..." %(date, event, fulljobid))
def parse_accounting_line(line, lineno):
""" Parse one line from accounting log and insert the data into DB.
"""
global server,last_event_time
logger = logging.getLogger("goove_updater")
cursor = connection.cursor()
try:
date,event,fulljobid,attrs = line.split(';')
except ValueError:
logger.warning("skipping invalid line %d: '%s'" % (lineno,line))
return
logger.debug("Processing accounting line: %s:%s:%s ..." %(date, event, fulljobid))
last_event_time = datetime.datetime.strptime(date, "%m/%d/%Y %H:%M:%S")
if last_event_time < server.lastacc_update:
logger.info("Ignoring accounting event with datetime %s" % str(last_event_time))
return
# We ignore PBSPro Licensing lines (it is not job related)
if event=='L':
logger.debug("Ignoring licensing line")
return
attrdir = {}
try:
for key,val in map(lambda x: x.split('=',1), attrs.split()):
attrdir[key] = val
except ValueError:
logger.warning("Skipping line with invalid attribues %d: '%s'" % (lineno,attrs))
jobid_name, server_name = JOBID_REGEX.search(fulljobid).groups()
if server_name!=server.name:
logger.warning("The name of the server in jobid: %s differs from server name in database: %s" % (server_name,server.name))
job = SQLJob()
job.jobid = jobid_name
job.server_id = server.id
job.refresh_id_jobstate_id()
if attrdir.has_key('owner'):
shname = attrdir['owner'].split('@')[1]
submithost,created = getSubmitHost(shname)
if created:
logger.info("new submit host will be created: %s" % shname)
job.submithost_id = submithost.pk
if attrdir.has_key('requestor'):
shname = attrdir['requestor'].split('@')[1]
submithost,created = getSubmitHost(shname)
if created:
logger.info("new submit host will be created: %s" % shname)
job.submithost_id = submithost.id
if attrdir.has_key('group'):
group,created = getGroup(attrdir['group'], server)
if created:
logger.info("new group will be created: %s" % attrdir['group'])
if attrdir.has_key('user'):
user,created = getUser(attrdir['user'], server, group)
if created:
logger.info("new user will be created: %s" % attrdir['user'])
job.job_owner_id = user.id
# TODO: convert this to direct SQL as well
user.group = group
if attrdir.has_key('resources_used.cput'):
h,m,s = attrdir['resources_used.cput'].split(":")
job.cput = (int(h)*60+int(m))*60+int(s)
if attrdir.has_key('resources_used.walltime'):
h,m,s = attrdir['resources_used.walltime'].split(":")
job.walltime = (int(h)*60+int(m))*60+int(s)
if attrdir.has_key('resources_used.cput') and attrdir.has_key('resources_used.walltime'):
if job.walltime!=0:
job.efficiency = 100*job.cput/job.walltime
else:
job.efficiency = 0
if attrdir.has_key('Exit_status'):
job.exit_status = int(attrdir['Exit_status'])
if event=='Q':
new_state = getJobState('Q')
elif event=='S' or event=='R' or event=='C' or event=='T':
new_state = getJobState('R')
elif event=='E':
new_state = getJobState('C')
elif event=='D':
new_state = getJobState('D')
elif event=='A':
new_state = getJobState('A')
elif event=='G':
new_state = getJobState('D')
else:
logger.error("Unknown event type in accounting log file: %s" % line)
return
if job.job_state_id != getJobState('C').id:
job.job_state_id = new_state.id
else:
logger.info("Job %s.%s is already finished, not changing the state." % (job.jobid,server.name))
if attrdir.has_key('queue'):
queue,created = getQueue(attrdir['queue'], server)
if created:
logger.info("new queue will be created: %s" % attrdir['queue'])
job.queue_id = queue.id
if attrdir.has_key('ctime'):
job.ctime = datetime.datetime.fromtimestamp(int(attrdir['ctime']))
if attrdir.has_key('mtime'):
job.mtime = datetime.datetime.fromtimestamp(int(attrdir['mtime']))
if attrdir.has_key('qtime'):
job.qtime = datetime.datetime.fromtimestamp(int(attrdir['qtime']))
if attrdir.has_key('etime'):
job.etime = datetime.datetime.fromtimestamp(int(attrdir['etime']))
if attrdir.has_key('start'):
job.start_time = datetime.datetime.fromtimestamp(int(attrdir['start']))
if attrdir.has_key('end'):
job.comp_time = datetime.datetime.fromtimestamp(int(attrdir['end']))
if attrdir.has_key('exec_host'):
exec_host_names_slots = attrdir['exec_host'].split('+')
job.jobslots = []
# convert PBSPro records like 'node1/0*2' to more generic 'node1/0+node1/1'
exec_host_names_slots_new = []
for exec_host_name_slot in exec_host_names_slots:
if exec_host_name_slot.find('*')>=0:
exec_host_slot0, numslots = exec_host_name_slot.split('*')
exec_host_name = exec_host_slot0.split('/')[0]
exec_host_name_slot_new=[ "%s/%d" % (exec_host_name, i) for i in range(0,int(numslots)) ]
logger.debug("Exec_host %s converted to %s" % (exec_host_name_slot,exec_host_name_slot_new))
exec_host_names_slots_new.extend(exec_host_name_slot_new)
else:
exec_host_names_slots_new.append(exec_host_name_slot)
exec_host_names_slots = exec_host_names_slots_new
for exec_host_name_slot in exec_host_names_slots:
name,slotstr = exec_host_name_slot.split('/')
slot = int(slotstr)
node,created = getNode(name, server)
if created:
logger.info("new node will be created: node name: %s" % (name))
node.save()
js,created = getJobSlot(slot=slot,node=node)
if created:
logger.info("new jobslot will be created: slot: %d, node name: %s" % (slot,name))
js.save()
job.jobslots.append(js.id)
job.save()
if job.id == -1:
job.refresh_id_jobstate_id()
d,t = date.split(' ')
m,d,y = d.split('/')
timestamp='%s-%s-%s %s' % (y,m,d,t)
cursor.execute("INSERT IGNORE INTO trqacc_accountingevent (timestamp, type, job_id) VALUES (%s,%s,%s)", [timestamp, event, job.id])
cursor.execute("SELECT LAST_INSERT_ID()")
row = cursor.fetchone()
ae_id = row[0]
# already existing accounting event?
if ae_id > 0:
for key,val in attrdir.items():
ea,created = getEventAttribute(key)
cursor.execute("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)", [ae_id, ea.id, val])
