def _process_event(self, id_, event):
"""Processes the given event, updating the instance data
structures accordingly."""
datetime_ = event['datetime']
if event['status'] is not None:
status = event['status']
prevstatus = self.status[id_]['status']
# Avoid overwriting a status with a less important one.
if status == CrabStatus.CLEARED:
self.status[id_]['status'] = status
elif CrabStatus.is_trivial(status):
if prevstatus is None or CrabStatus.is_ok(prevstatus):
self.status[id_]['status'] = status
elif CrabStatus.is_warning(status):
if prevstatus is None or not CrabStatus.is_error(prevstatus):
self.status[id_]['status'] = status
# Always set success / failure status (the remaining options).
else:
self.status[id_]['status'] = status
if not CrabStatus.is_trivial(status):
history = self.status[id_]['history']
if len(history) >= HISTORY_COUNT:
del history[0]
history.append(status)
# Handle ALREADYRUNNING as a 'start' type event, so that
# the MISSED alarm is not raised and the timeout period
# is extended.
if (event['type'] == CrabEvent.START or
event['status'] == CrabStatus.ALREADYRUNNING):
self.status[id_]['running'] = True
if not self.passive:
self.last_start[id_] = datetime_
self.timeout[id_] = datetime_ + self.config[id_]['timeout']
if id_ in self.late_timeout:
del self.late_timeout[id_]
if id_ in self.miss_timeout:
del self.miss_timeout[id_]
elif (event['type'] == CrabEvent.FINISH or
event['status'] == CrabStatus.TIMEOUT):
self.status[id_]['running'] = False
if not self.passive:
if id_ in self.timeout:
del self.timeout[id_]
def _process_event(self, id_, event):
"""Processes the given event, updating the instance data
structures accordingly."""
datetime_ = event['datetime']
if event['status'] is not None:
status = event['status']
prevstatus = self.status[id_]['status']
# Avoid overwriting a status with a less important one.
if status == CrabStatus.CLEARED:
self.status[id_]['status'] = status
elif CrabStatus.is_trivial(status):
if prevstatus is None or CrabStatus.is_ok(prevstatus):
self.status[id_]['status'] = status
elif CrabStatus.is_warning(status):
if prevstatus is None or not CrabStatus.is_error(prevstatus):
self.status[id_]['status'] = status
# Always set success / failure status (the remaining options).
else:
self.status[id_]['status'] = status
if not CrabStatus.is_trivial(status):
history = self.status[id_]['history']
if len(history) >= HISTORY_COUNT:
del history[0]
history.append(status)
# Handle ALREADYRUNNING as a 'start' type event, so that
# the MISSED alarm is not raised and the timeout period
# is extended.
if (event['type'] == CrabEvent.START
or event['status'] == CrabStatus.ALREADYRUNNING):
self.status[id_]['running'] = True
if not self.passive:
self.last_start[id_] = datetime_
self.timeout[id_] = datetime_ + self.config[id_]['timeout']
if id_ in self.late_timeout:
del self.late_timeout[id_]
if id_ in self.miss_timeout:
del self.miss_timeout[id_]
elif (event['type'] == CrabEvent.FINISH
or event['status'] == CrabStatus.TIMEOUT):
self.status[id_]['running'] = False
if not self.passive:
if id_ in self.timeout:
del self.timeout[id_]
def __call__(self,
events,
skip_ok=False,
skip_warning=False,
skip_error=False,
skip_trivial=True,
skip_start=False,
squash_start=False):
"""Performs filtering, and returns the altered event list."""
output = []
squash = set()
self.errors = 0
self.warnings = 0
for (i, e) in enumerate(events):
if i in squash:
continue
e = e.copy()
if e['type'] == CrabEvent.START:
if skip_start:
continue
else:
if (skip_trivial and CrabStatus.is_trivial(e['status'])
or skip_ok and CrabStatus.is_ok(e['status'])
or skip_warning and CrabStatus.is_warning(e['status'])
or skip_error and CrabStatus.is_error(e['status'])):
continue
if CrabStatus.is_error(e['status']):
self.errors += 1
if CrabStatus.is_warning(e['status']):
self.warnings += 1
if squash_start and e['type'] == CrabEvent.FINISH:
start = _find_previous_start(events, i)
if start is not None:
squash.add(start)
delta = (
self.store.parse_datetime(e['datetime']) -
self.store.parse_datetime(events[start]['datetime']))
e['duration'] = str(delta)
e['datetime'] = self.in_timezone(e['datetime'])
output.append(e)
return output
def __call__(self, events, skip_ok=False, skip_warning=False,
skip_error=False, skip_trivial=True, skip_start=False,
squash_start=False):
"""Performs filtering, and returns the altered event list."""
output = []
squash = set()
self.errors = 0
self.warnings = 0
for (i, e) in enumerate(events):
if i in squash:
continue
e = e.copy()
if e['type'] == CrabEvent.START:
if skip_start:
continue
else:
if (skip_trivial and CrabStatus.is_trivial(e['status'])
or skip_ok and CrabStatus.is_ok(e['status'])
or skip_warning and CrabStatus.is_warning(e['status'])
or skip_error and CrabStatus.is_error(e['status'])):
continue
if CrabStatus.is_error(e['status']):
self.errors += 1
if CrabStatus.is_warning(e['status']):
self.warnings += 1
if squash_start and e['type'] == CrabEvent.FINISH:
start = _find_previous_start(events, i)
if start is not None:
squash.add(start)
delta = (self.store.parse_datetime(e['datetime'])
- self.store.parse_datetime(events[start]['datetime']))
e['duration'] = str(delta)
e['datetime'] = self.in_timezone(e['datetime'])
output.append(e)
return output
def run(self):
"""Monitor thread main run function.
When the thread is started, this function will run. It begins
by fetching a list of jobs and using them to populate its
data structures. When this is complete, the Event status_ready
is fired.
It then goes into a loop, and every few seconds it checks
for new events, processing any which are found. The new_event
Condition is fired if there were any new events.
We call _check_minute from CrabMinutely to check whether the
minute has changed since the last time round the loop."""
jobs = self.store.get_jobs()
for job in jobs:
id_ = job['id']
try:
self._initialize_job(id_, load_events=True)
except JobDeleted:
logger.warning('Warning: job {} has vanished'.format(id_))
self.status_ready.set()
while True:
time.sleep(5)
datetime_ = datetime.now(pytz.UTC)
# Retrieve events. Trap exceptions in case of database
# disconnection.
events = []
try:
events = self.store.get_events_since(self.max_startid,
self.max_alarmid,
self.max_finishid)
except Exception as e:
logger.exception('Error: monitor exception getting events')
for event in events:
id_ = event['jobid']
self._update_max_id_values(event)
try:
if id_ not in self.status:
self._initialize_job(id_)
self._process_event(id_, event)
self._compute_reliability(id_)
# If the monitor is loaded when a job has just been
# deleted, then it may have events more recent
# than those of the events that still exist.
except JobDeleted:
pass
# Also trap other exceptions, in case a database disconnection
# causes a failure from _initialize_job. Do this separately,
# inside the events loop so that we keep the max_id_values
# up to date with the other events.
except Exception as e:
logger.exception('Error: monitor exception handling event')
self.num_error = 0
self.num_warning = 0
for id_ in self.status:
jobstatus = self.status[id_]['status']
if (jobstatus is None or CrabStatus.is_ok(jobstatus)):
pass
elif (CrabStatus.is_warning(jobstatus)):
self.num_warning += 1
else:
self.num_error += 1
if events:
with self.new_event:
self.new_event.notify_all()
# Allow superclass CrabMinutely to call our run_minutely
# method as required. Note: the call back to run_minutely
# is protected by a try-except block in the superclass.
self._check_minute()
# Check status of timeouts - need to get a list of keys
# so that we can delete from the dict while iterating.
# Note: _write_alarm uses a try-except block for CrabErrors.
for id_ in list(self.late_timeout.keys()):
if self.late_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.LATE)
del self.late_timeout[id_]
for id_ in list(self.miss_timeout.keys()):
if self.miss_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.MISSED)
del self.miss_timeout[id_]
for id_ in list(self.timeout.keys()):
if self.timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.TIMEOUT)
del self.timeout[id_]
def run(self):
"""Monitor thread main run function.
When the thread is started, this function will run. It begins
by fetching a list of jobs and using them to populate its
data structures. When this is complete, the Event status_ready
is fired.
It then goes into a loop, and every few seconds it checks
for new events, processing any which are found. The new_event
Condition is fired if there were any new events.
We call _check_minute from CrabMinutely to check whether the
minute has changed since the last time round the loop."""
jobs = self.store.get_jobs()
for job in jobs:
id_ = job['id']
try:
self._initialize_job(id_, load_events=True)
except JobDeleted:
print('Warning: job', id_, 'has vanished')
self.status_ready.set()
while True:
time.sleep(5)
datetime_ = datetime.datetime.now(pytz.UTC)
events = self.store.get_events_since(self.max_startid,
self.max_alarmid,
self.max_finishid)
for event in events:
id_ = event['jobid']
self._update_max_id_values(event)
try:
if id_ not in self.status:
self._initialize_job(id_)
self._process_event(id_, event)
self._compute_reliability(id_)
# If the monitor is loaded when a job has just been
# deleted, then it may have events more recent
# than those of the events that still exist.
except JobDeleted:
pass
self.num_error = 0
self.num_warning = 0
for id_ in self.status:
jobstatus = self.status[id_]['status']
if (jobstatus is None or CrabStatus.is_ok(jobstatus)):
pass
elif (CrabStatus.is_warning(jobstatus)):
self.num_warning += 1
else:
self.num_error += 1
if events:
with self.new_event:
self.new_event.notify_all()
# Allow superclass CrabMinutely to call our run_minutely
# method as required.
self._check_minute()
# Check status of timeouts - need to get a list of keys
# so that we can delete from the dict while iterating.
for id_ in list(self.miss_timeout.keys()):
if self.miss_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.MISSED)
del self.miss_timeout[id_]
for id_ in list(self.timeout.keys()):
if self.timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.TIMEOUT)
del self.timeout[id_]
def run(self):
"""Monitor thread main run function.
When the thread is started, this function will run. It begins
by fetching a list of jobs and using them to populate its
data structures. When this is complete, the Event status_ready
is fired.
It then goes into a loop, and every few seconds it checks
for new events, processing any which are found. The new_event
Condition is fired if there were any new events.
We call _check_minute from CrabMinutely to check whether the
minute has changed since the last time round the loop."""
jobs = self.store.get_jobs()
for job in jobs:
id_ = job['id']
try:
self._initialize_job(id_, load_events=True)
except JobDeleted:
logger.warning('Warning: job {} has vanished'.format(id_))
self.status_ready.set()
while True:
time.sleep(5)
datetime_ = datetime.now(pytz.UTC)
# Retrieve events. Trap exceptions in case of database
# disconnection.
events = []
try:
events = self.store.get_events_since(
self.max_startid, self.max_alarmid, self.max_finishid)
except Exception as e:
logger.exception('Error: monitor exception getting events')
for event in events:
id_ = event['jobid']
self._update_max_id_values(event)
try:
if id_ not in self.status:
self._initialize_job(id_)
self._process_event(id_, event)
self._compute_reliability(id_)
# If the monitor is loaded when a job has just been
# deleted, then it may have events more recent
# than those of the events that still exist.
except JobDeleted:
pass
# Also trap other exceptions, in case a database disconnection
# causes a failure from _initialize_job. Do this separately,
# inside the events loop so that we keep the max_id_values
# up to date with the other events.
except Exception as e:
logger.exception('Error: monitor exception handling event')
self.num_error = 0
self.num_warning = 0
for id_ in self.status:
jobstatus = self.status[id_]['status']
if (jobstatus is None or CrabStatus.is_ok(jobstatus)):
pass
elif (CrabStatus.is_warning(jobstatus)):
self.num_warning += 1
else:
self.num_error += 1
if events:
with self.new_event:
self.new_event.notify_all()
# Allow superclass CrabMinutely to call our run_minutely
# method as required. Note: the call back to run_minutely
# is protected by a try-except block in the superclass.
self._check_minute()
# Check status of timeouts - need to get a list of keys
# so that we can delete from the dict while iterating.
# Note: _write_alarm uses a try-except block for CrabErrors.
for id_ in list(self.late_timeout.keys()):
if self.late_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.LATE)
del self.late_timeout[id_]
for id_ in list(self.miss_timeout.keys()):
if self.miss_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.MISSED)
del self.miss_timeout[id_]
for id_ in list(self.timeout.keys()):
if self.timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.TIMEOUT)
del self.timeout[id_]
def run(self):
"""Monitor thread main run function.
When the thread is started, this function will run. It begins
by fetching a list of jobs and using them to populate its
data structures. When this is complete, the Event status_ready
is fired.
It then goes into a loop, and every few seconds it checks
for new events, processing any which are found. The new_event
Condition is fired if there were any new events.
We call _check_minute from CrabMinutely to check whether the
minute has changed since the last time round the loop."""
jobs = self.store.get_jobs()
for job in jobs:
id_ = job['id']
try:
self._initialize_job(id_, load_events=True)
except JobDeleted:
print('Warning: job', id_, 'has vanished')
self.status_ready.set()
while True:
time.sleep(5)
datetime_ = datetime.datetime.now(pytz.UTC)
events = self.store.get_events_since(self.max_startid,
self.max_alarmid, self.max_finishid)
for event in events:
id_ = event['jobid']
self._update_max_id_values(event)
try:
if id_ not in self.status:
self._initialize_job(id_)
self._process_event(id_, event)
self._compute_reliability(id_)
# If the monitor is loaded when a job has just been
# deleted, then it may have events more recent
# than those of the events that still exist.
except JobDeleted:
pass
self.num_error = 0;
self.num_warning = 0;
for id_ in self.status:
jobstatus = self.status[id_]['status']
if (jobstatus is None or CrabStatus.is_ok(jobstatus)):
pass
elif (CrabStatus.is_warning(jobstatus)):
self.num_warning += 1;
else:
self.num_error += 1;
if events:
with self.new_event:
self.new_event.notify_all()
# Allow superclass CrabMinutely to call our run_minutely
# method as required.
self._check_minute()
# Check status of timeouts - need to get a list of keys
# so that we can delete from the dict while iterating.
for id_ in list(self.miss_timeout.keys()):
if self.miss_timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.MISSED)
del self.miss_timeout[id_]
for id_ in list(self.timeout.keys()):
if self.timeout[id_] < datetime_:
self._write_alarm(id_, CrabStatus.TIMEOUT)
del self.timeout[id_]
