def __init__(self, submit_dir): ''' initialize - connect to the database. ''' logger.debug ("event-scanner:<init> submit_dir: %s", submit_dir) properties = os.path.join (submit_dir, "grayson.stampede.properties") properties = None self.db_url, self.wf_uuid = db_utils.get_db_url_wf_uuid (submit_dir, properties) logger.debug ("dburl=(%s) wf_uuid=(%s)", self.db_url, self.wf_uuid) self.database = StampedeStatistics (self.db_url, True) self.database.initialize (self.wf_uuid) self.totalMessages = 0
class EventScanner (object): ''' Stampede event scanner ''' DAX_START = "dax-start" DAX_END = "dax-end" def __init__(self, submit_dir): ''' initialize - connect to the database. ''' logger.debug ("event-scanner:<init> submit_dir: %s", submit_dir) properties = os.path.join (submit_dir, "grayson.stampede.properties") properties = None self.db_url, self.wf_uuid = db_utils.get_db_url_wf_uuid (submit_dir, properties) logger.debug ("dburl=(%s) wf_uuid=(%s)", self.db_url, self.wf_uuid) self.database = StampedeStatistics (self.db_url, True) self.database.initialize (self.wf_uuid) self.totalMessages = 0 def createWorkflowEvent (self, record): ''' Map a result set into an event object. ''' event = WorkflowEvent () event.name = record.exec_job_id event.is_dax = record.exec_job_id.find ('subdax_') == 0 event.site = record.site event.timestamp = record.timestamp event.exitcode = utils.raw_to_regular (record.exitcode) event.hostname = record.hostname event.sched_id = record.sched_id event.work_dir = os.path.abspath (record.work_dir) event.work_dir = event.work_dir.replace ("work/outputs", "work") event.dax_file = os.path.basename (record.dax_file) #os.path.basename (record.dax_file) if record.dax_file else StatsUtil.formDaxName (event.workdir) event.state = record.state event.work_dir = record.work_dir event.stdout = record.stdout_file event.stderr = record.stderr_file #logger.debug (json.dumps (event, sort_keys=True, indent=3, cls=WorkflowEventEncoder)) return event ''' SELECT job.job_id AS job_job_id, job.exec_job_id AS job_exec_job_id, job_instance.site AS job_instance_site, job_instance.sched_id AS job_instance_sched_id, job_instance.stdout_file AS job_instance_stdout_file, job_instance.stderr_file AS job_instance_stderr_file, job_instance.exitcode AS job_instance_exitcode, CAST(jobstate.timestamp AS FLOAT) AS TIMESTAMP, jobstate.state AS jobstate_state, host.hostname AS host_hostname, workflow.user AS workflow_user, workflow.dax_file AS workflow_dax_file, workflow.submit_dir AS work_dir FROM job, job_instance, jobstate, HOST, workflow WHERE job.job_id = job_instance.job_id AND job_instance.host_id = HOST.host_id AND job_instance.job_instance_id = jobstate.job_instance_id AND job.wf_id = workflow.wf_id AND jobstate.TIMESTAMP > ? AND NOT ((job_instance.exitcode = ? OR job_instance.exitcode = ?) AND (jobstate.state = ? OR jobstate.state = ?)) AND (job.exec_job_id NOT LIKE ? OR job.exec_job_id LIKE ?) AND workflow.dax_file LIKE ? AND job.exec_job_id NOT LIKE ? AND job.exec_job_id NOT LIKE ? AND job.exec_job_id NOT LIKE ? AND job.exec_job_id NOT LIKE ? ORDER BY jobstate.TIMESTAMP ''' def selectWorkflowEvents (self, since = 0, daxen = {}, jobFilter = None): ''' Select events from the stampede schema. ''' query = self.database.session.query (Job.job_id, Job.exec_job_id, JobInstance.site, JobInstance.sched_id, JobInstance.stdout_file, JobInstance.stderr_file, JobInstance.exitcode, cast (Jobstate.timestamp, Float).label ('timestamp'), Jobstate.state, Host.hostname, Workflow.user, Workflow.dax_file, Workflow.submit_dir.label ('work_dir')) # unfortunate name mismatch between schema and existing code. query = query.filter (Job.job_id == JobInstance.job_id, JobInstance.host_id == Host.host_id, JobInstance.job_instance_id == Jobstate.job_instance_id, Job.wf_id == Workflow.wf_id, Jobstate.timestamp > since) query = query.order_by (JobInstance.sched_id, Jobstate.timestamp) ''' don't include intermediate statuses ''' query = query.filter (not_(and_(or_(JobInstance.exitcode == 0, JobInstance.exitcode == 1), or_(Jobstate.state == 'SUBMIT', Jobstate.state == 'EXECUTE')))) ''' ensure that either (a) the event does not pertain to a subdax or that, (b) if it does, the subdax is one in the list of daxen ''' filters = [] for dax in daxen: filters.append (Job.exec_job_id.like ('subdax_%s%%' % dax)) query = query.filter (or_( not_(Job.exec_job_id.like ('subdax_%')), or_(*tuple (filters)))) ''' ensure dax_file of this event matches one of the daxen in the filter ''' filters = [] for dax in daxen: filters.append (Workflow.dax_file.like ('%%%s%%' % dax)) query = query.filter (or_ (*tuple (filters))) ''' ignore job prefixes in the job filter. ''' if jobFilter: for pattern in jobFilter: query = query.filter (not_ (Job.exec_job_id.like ('%s%%' % pattern))) ''' return the cursor. ''' return query.all () def getWorkflowStatus (self): ''' Find out if the workflow is still running ''' return self.database.session.query (Workflowstate.status).filter (Workflow.wf_id == Workflowstate.wf_id, Workflow.wf_id == Workflow.root_wf_id, Workflowstate.state == 'WORKFLOW_TERMINATED').first () def getEvents (self, processor, wf_uuid = None, jobFilter = [ 'clean_up', 'chmod', 'register_', 'create_dir_' ], since = 0, daxen = {}): ''' Get events for the workflow since a given timestamp. ''' max_timestamp = since status = self.getWorkflowStatus () events = self.selectWorkflowEvents (since, daxen, jobFilter) count = 0 for event in events: processor.processEvent (self.createWorkflowEvent (event)) max_timestamp = max (max_timestamp, event.timestamp) count += 1 self.totalMessages += count if logger.isEnabledFor (logging.DEBUG): logger.debug ("=====================================================================================================") logger.debug ("== events: cycle=%s total=%s", count, self.totalMessages) logger.debug ("== total events: %s ", count) logger.debug ("== max_timestamp: %s ", max_timestamp) logger.debug ("=====================================================================================================") return status, max_timestamp def emitJSON (self, obj): return json.dumps (obj, cls = WorkflowEventEncoder, indent = 2, sort_keys = True)