def parse_accounting_line(line, lineno): """ Parse one line from accounting log and insert the data into DB. """ global server,last_event_time logger = logging.getLogger("goove_updater") cursor = connection.cursor() try: date,event,fulljobid,attrs = line.split(';') except ValueError: logger.warning("skipping invalid line %d: '%s'" % (lineno,line)) return logger.debug("Processing accounting line: %s:%s:%s ..." %(date, event, fulljobid)) last_event_time = datetime.datetime.strptime(date, "%m/%d/%Y %H:%M:%S") if last_event_time < server.lastacc_update: logger.info("Ignoring accounting event with datetime %s" % str(last_event_time)) return # We ignore PBSPro Licensing lines (it is not job related) if event=='L': logger.debug("Ignoring licensing line") return attrdir = {} try: for key,val in map(lambda x: x.split('=',1), attrs.split()): attrdir[key] = val except ValueError: logger.warning("Skipping line with invalid attribues %d: '%s'" % (lineno,attrs)) jobid_name, server_name = JOBID_REGEX.search(fulljobid).groups() if server_name!=server.name: logger.warning("The name of the server in jobid: %s differs from server name in database: %s" % (server_name,server.name)) d,t = date.split(' ') m,d,y = d.split('/') timestamp='%s-%s-%s %s' % (y,m,d,t) retval = cursor.execute("INSERT IGNORE INTO trqacc_accountingevent (timestamp, type, short_jobid, server_id) VALUES (%s,%s,%s,%s)", [timestamp, event, jobid_name, server.id]) if retval==1: # FIXME: I am not sure this is thread safe - should call something like C-API mysql_insert_id(); cursor.execute("SELECT LAST_INSERT_ID()") row = cursor.fetchone() ae_id = row[0] for key,val in attrdir.items(): ea,created = getEventAttribute(key) logger.debug("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)" % (ae_id, ea.id, val)) cursor.execute("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)", [ae_id, ea.id, val]) else: logger.warning("Tried to insert already present accounting event - skipping, check your logs for: %s:%s:%s ..." %(date, event, fulljobid))
def parse_accounting_line(line, lineno): """ Parse one line from accounting log and insert the data into DB. """ global server,last_event_time logger = logging.getLogger("goove_updater") cursor = connection.cursor() try: date,event,fulljobid,attrs = line.split(';') except ValueError: logger.warning("skipping invalid line %d: '%s'" % (lineno,line)) return logger.debug("Processing accounting line: %s:%s:%s ..." %(date, event, fulljobid)) last_event_time = datetime.datetime.strptime(date, "%m/%d/%Y %H:%M:%S") if last_event_time < server.lastacc_update: logger.info("Ignoring accounting event with datetime %s" % str(last_event_time)) return # We ignore PBSPro Licensing lines (it is not job related) if event=='L': logger.debug("Ignoring licensing line") return attrdir = {} try: for key,val in map(lambda x: x.split('=',1), attrs.split()): attrdir[key] = val except ValueError: logger.warning("Skipping line with invalid attribues %d: '%s'" % (lineno,attrs)) jobid_name, server_name = JOBID_REGEX.search(fulljobid).groups() if server_name!=server.name: logger.warning("The name of the server in jobid: %s differs from server name in database: %s" % (server_name,server.name)) job = SQLJob() job.jobid = jobid_name job.server_id = server.id job.refresh_id_jobstate_id() if attrdir.has_key('owner'): shname = attrdir['owner'].split('@')[1] submithost,created = getSubmitHost(shname) if created: logger.info("new submit host will be created: %s" % shname) job.submithost_id = submithost.pk if attrdir.has_key('requestor'): shname = attrdir['requestor'].split('@')[1] submithost,created = getSubmitHost(shname) if created: logger.info("new submit host will be created: %s" % shname) job.submithost_id = submithost.id if attrdir.has_key('group'): group,created = getGroup(attrdir['group'], server) if created: logger.info("new group will be created: %s" % attrdir['group']) if attrdir.has_key('user'): user,created = getUser(attrdir['user'], server, group) if created: logger.info("new user will be created: %s" % attrdir['user']) job.job_owner_id = user.id # TODO: convert this to direct SQL as well user.group = group if attrdir.has_key('resources_used.cput'): h,m,s = attrdir['resources_used.cput'].split(":") job.cput = (int(h)*60+int(m))*60+int(s) if attrdir.has_key('resources_used.walltime'): h,m,s = attrdir['resources_used.walltime'].split(":") job.walltime = (int(h)*60+int(m))*60+int(s) if attrdir.has_key('resources_used.cput') and attrdir.has_key('resources_used.walltime'): if job.walltime!=0: job.efficiency = 100*job.cput/job.walltime else: job.efficiency = 0 if attrdir.has_key('Exit_status'): job.exit_status = int(attrdir['Exit_status']) if event=='Q': new_state = getJobState('Q') elif event=='S' or event=='R' or event=='C' or event=='T': new_state = getJobState('R') elif event=='E': new_state = getJobState('C') elif event=='D': new_state = getJobState('D') elif event=='A': new_state = getJobState('A') elif event=='G': new_state = getJobState('D') else: logger.error("Unknown event type in accounting log file: %s" % line) return if job.job_state_id != getJobState('C').id: job.job_state_id = new_state.id else: logger.info("Job %s.%s is already finished, not changing the state." % (job.jobid,server.name)) if attrdir.has_key('queue'): queue,created = getQueue(attrdir['queue'], server) if created: logger.info("new queue will be created: %s" % attrdir['queue']) job.queue_id = queue.id if attrdir.has_key('ctime'): job.ctime = datetime.datetime.fromtimestamp(int(attrdir['ctime'])) if attrdir.has_key('mtime'): job.mtime = datetime.datetime.fromtimestamp(int(attrdir['mtime'])) if attrdir.has_key('qtime'): job.qtime = datetime.datetime.fromtimestamp(int(attrdir['qtime'])) if attrdir.has_key('etime'): job.etime = datetime.datetime.fromtimestamp(int(attrdir['etime'])) if attrdir.has_key('start'): job.start_time = datetime.datetime.fromtimestamp(int(attrdir['start'])) if attrdir.has_key('end'): job.comp_time = datetime.datetime.fromtimestamp(int(attrdir['end'])) if attrdir.has_key('exec_host'): exec_host_names_slots = attrdir['exec_host'].split('+') job.jobslots = [] # convert PBSPro records like 'node1/0*2' to more generic 'node1/0+node1/1' exec_host_names_slots_new = [] for exec_host_name_slot in exec_host_names_slots: if exec_host_name_slot.find('*')>=0: exec_host_slot0, numslots = exec_host_name_slot.split('*') exec_host_name = exec_host_slot0.split('/')[0] exec_host_name_slot_new=[ "%s/%d" % (exec_host_name, i) for i in range(0,int(numslots)) ] logger.debug("Exec_host %s converted to %s" % (exec_host_name_slot,exec_host_name_slot_new)) exec_host_names_slots_new.extend(exec_host_name_slot_new) else: exec_host_names_slots_new.append(exec_host_name_slot) exec_host_names_slots = exec_host_names_slots_new for exec_host_name_slot in exec_host_names_slots: name,slotstr = exec_host_name_slot.split('/') slot = int(slotstr) node,created = getNode(name, server) if created: logger.info("new node will be created: node name: %s" % (name)) node.save() js,created = getJobSlot(slot=slot,node=node) if created: logger.info("new jobslot will be created: slot: %d, node name: %s" % (slot,name)) js.save() job.jobslots.append(js.id) job.save() if job.id == -1: job.refresh_id_jobstate_id() d,t = date.split(' ') m,d,y = d.split('/') timestamp='%s-%s-%s %s' % (y,m,d,t) cursor.execute("INSERT IGNORE INTO trqacc_accountingevent (timestamp, type, job_id) VALUES (%s,%s,%s)", [timestamp, event, job.id]) cursor.execute("SELECT LAST_INSERT_ID()") row = cursor.fetchone() ae_id = row[0] # already existing accounting event? if ae_id > 0: for key,val in attrdir.items(): ea,created = getEventAttribute(key) cursor.execute("INSERT INTO trqacc_eventattributevalue (ae_id, ea_id, value) VALUES (%s,%s,%s)", [ae_id, ea.id, val])