def __init__(self, submit_dir):
''' initialize - connect to the database. '''
logger.debug ("event-scanner:<init> submit_dir: %s", submit_dir)
properties = os.path.join (submit_dir, "grayson.stampede.properties")
properties = None
self.db_url, self.wf_uuid = db_utils.get_db_url_wf_uuid (submit_dir, properties)
logger.debug ("dburl=(%s) wf_uuid=(%s)", self.db_url, self.wf_uuid)
self.database = StampedeStatistics (self.db_url, True)
self.database.initialize (self.wf_uuid)
self.totalMessages = 0
class EventScanner (object):
''' Stampede event scanner '''
DAX_START = "dax-start"
DAX_END = "dax-end"
def __init__(self, submit_dir):
''' initialize - connect to the database. '''
logger.debug ("event-scanner:<init> submit_dir: %s", submit_dir)
properties = os.path.join (submit_dir, "grayson.stampede.properties")
properties = None
self.db_url, self.wf_uuid = db_utils.get_db_url_wf_uuid (submit_dir, properties)
logger.debug ("dburl=(%s) wf_uuid=(%s)", self.db_url, self.wf_uuid)
self.database = StampedeStatistics (self.db_url, True)
self.database.initialize (self.wf_uuid)
self.totalMessages = 0
def createWorkflowEvent (self, record):
''' Map a result set into an event object. '''
event = WorkflowEvent ()
event.name = record.exec_job_id
event.is_dax = record.exec_job_id.find ('subdax_') == 0
event.site = record.site
event.timestamp = record.timestamp
event.exitcode = utils.raw_to_regular (record.exitcode)
event.hostname = record.hostname
event.sched_id = record.sched_id
event.work_dir = os.path.abspath (record.work_dir)
event.work_dir = event.work_dir.replace ("work/outputs", "work")
event.dax_file = os.path.basename (record.dax_file) #os.path.basename (record.dax_file) if record.dax_file else StatsUtil.formDaxName (event.workdir)
event.state = record.state
event.work_dir = record.work_dir
event.stdout = record.stdout_file
event.stderr = record.stderr_file
#logger.debug (json.dumps (event, sort_keys=True, indent=3, cls=WorkflowEventEncoder))
return event
'''
SELECT job.job_id AS job_job_id,
job.exec_job_id AS job_exec_job_id,
job_instance.site AS job_instance_site,
job_instance.sched_id AS job_instance_sched_id,
job_instance.stdout_file AS job_instance_stdout_file,
job_instance.stderr_file AS job_instance_stderr_file,
job_instance.exitcode AS job_instance_exitcode,
CAST(jobstate.timestamp AS FLOAT) AS TIMESTAMP,
jobstate.state AS jobstate_state,
host.hostname AS host_hostname,
workflow.user AS workflow_user,
workflow.dax_file AS workflow_dax_file,
workflow.submit_dir AS work_dir
FROM job, job_instance, jobstate, HOST, workflow
WHERE job.job_id = job_instance.job_id
AND job_instance.host_id = HOST.host_id
AND job_instance.job_instance_id = jobstate.job_instance_id
AND job.wf_id = workflow.wf_id
AND jobstate.TIMESTAMP > ?
AND NOT ((job_instance.exitcode = ?
OR job_instance.exitcode = ?)
AND (jobstate.state = ?
OR jobstate.state = ?))
AND (job.exec_job_id NOT LIKE ?
OR job.exec_job_id LIKE ?)
AND workflow.dax_file LIKE ?
AND job.exec_job_id NOT LIKE ?
AND job.exec_job_id NOT LIKE ?
AND job.exec_job_id NOT LIKE ?
AND job.exec_job_id NOT LIKE ?
ORDER BY jobstate.TIMESTAMP
'''
def selectWorkflowEvents (self, since = 0, daxen = {}, jobFilter = None):
''' Select events from the stampede schema. '''
query = self.database.session.query (Job.job_id,
Job.exec_job_id,
JobInstance.site,
JobInstance.sched_id,
JobInstance.stdout_file,
JobInstance.stderr_file,
JobInstance.exitcode,
cast (Jobstate.timestamp, Float).label ('timestamp'),
Jobstate.state,
Host.hostname,
Workflow.user,
Workflow.dax_file,
Workflow.submit_dir.label ('work_dir')) # unfortunate name mismatch between schema and existing code.
query = query.filter (Job.job_id == JobInstance.job_id,
JobInstance.host_id == Host.host_id,
JobInstance.job_instance_id == Jobstate.job_instance_id,
Job.wf_id == Workflow.wf_id,
Jobstate.timestamp > since)
query = query.order_by (JobInstance.sched_id, Jobstate.timestamp)
''' don't include intermediate statuses '''
query = query.filter (not_(and_(or_(JobInstance.exitcode == 0, JobInstance.exitcode == 1),
or_(Jobstate.state == 'SUBMIT', Jobstate.state == 'EXECUTE'))))
''' ensure that either (a) the event does not pertain to a subdax or that,
(b) if it does, the subdax is one in the list of daxen '''
filters = []
for dax in daxen:
filters.append (Job.exec_job_id.like ('subdax_%s%%' % dax))
query = query.filter (or_( not_(Job.exec_job_id.like ('subdax_%')),
or_(*tuple (filters))))
''' ensure dax_file of this event matches one of the daxen in the filter '''
filters = []
for dax in daxen:
filters.append (Workflow.dax_file.like ('%%%s%%' % dax))
query = query.filter (or_ (*tuple (filters)))
''' ignore job prefixes in the job filter. '''
if jobFilter:
for pattern in jobFilter:
query = query.filter (not_ (Job.exec_job_id.like ('%s%%' % pattern)))
''' return the cursor. '''
return query.all ()
def getWorkflowStatus (self):
''' Find out if the workflow is still running '''
return self.database.session.query (Workflowstate.status).filter (Workflow.wf_id == Workflowstate.wf_id,
Workflow.wf_id == Workflow.root_wf_id,
Workflowstate.state == 'WORKFLOW_TERMINATED').first ()
def getEvents (self,
processor,
wf_uuid = None,
jobFilter = [ 'clean_up', 'chmod', 'register_', 'create_dir_' ],
since = 0,
daxen = {}):
''' Get events for the workflow since a given timestamp. '''
max_timestamp = since
status = self.getWorkflowStatus ()
events = self.selectWorkflowEvents (since, daxen, jobFilter)
count = 0
for event in events:
processor.processEvent (self.createWorkflowEvent (event))
max_timestamp = max (max_timestamp, event.timestamp)
count += 1
self.totalMessages += count
if logger.isEnabledFor (logging.DEBUG):
logger.debug ("=====================================================================================================")
logger.debug ("== events: cycle=%s total=%s", count, self.totalMessages)
logger.debug ("== total events: %s ", count)
logger.debug ("== max_timestamp: %s ", max_timestamp)
logger.debug ("=====================================================================================================")
return status, max_timestamp
def emitJSON (self, obj):
return json.dumps (obj,
cls = WorkflowEventEncoder,
indent = 2,
sort_keys = True)