def __init__(self, name, msg_target=None):
''' Initialize the checkpoint
Need the ruleset to enable getting to the current pool
'''
# setup the message target information
if msg_target is None:
# This only happens when testing
self.msg_target = MsgTargetLogger(prefix='TESTING: ')
else:
self.msg_target = msg_target
self.msg_target.debug('Creating IncidentPoolEventCheckpoint {0}'.format(name))
EventAnalyzerCheckpoint.__init__(self, name)
if self.data is None:
# No data so know pool end was the start
self.pool_rec_id = self.start_rec_id
else:
try:
self.pool_rec_id = long(json.loads(self.data)[1])
except ThreadKilled:
raise
except:
self.msg_target.warning('Checkpoint data invalid')
self.pool_rec_id = self.start_rec_id
self.data = None
self.rec_ids = self._get_rec_ids_from_data()
self.prime_incidents = []
return
class IncidentPoolEventCheckpoint(EventAnalyzerCheckpoint):
''' Handle checkpointing for an incident pool
self.start_rec_id is the min( last_in_pool, (all priming event rec_ids - 1))
data is a list of:
-- pool start time
-- last in pool
-- list of primes
-- compacted entry <rec_id><U|S><time_left>
-- optional +<dur_ext_time>
Note that if there is no primes then the start_rec_id is the last in pool so data is not recorded and
the start_rec_id is used as the last in pool rec_id
'''
__metaclass__ = ABCMeta
def __init__(self, name, msg_target=None):
''' Initialize the checkpoint
Need the ruleset to enable getting to the current pool
'''
# setup the message target information
if msg_target is None:
# This only happens when testing
self.msg_target = MsgTargetLogger(prefix='TESTING: ')
else:
self.msg_target = msg_target
self.msg_target.debug('Creating IncidentPoolEventCheckpoint {0}'.format(name))
EventAnalyzerCheckpoint.__init__(self, name)
if self.data is None:
# No data so know pool end was the start
self.pool_rec_id = self.start_rec_id
else:
try:
self.pool_rec_id = long(json.loads(self.data)[1])
except ThreadKilled:
raise
except:
self.msg_target.warning('Checkpoint data invalid')
self.pool_rec_id = self.start_rec_id
self.data = None
self.rec_ids = self._get_rec_ids_from_data()
self.prime_incidents = []
return
@abstractmethod
def get_pool(self):
''' Get the pool
Allows subclasses to control where the pool is managed
'''
pass
def need_to_analyze(self, event):
''' If before my checkpointed rec_id then don't need to process '''
chk_rec_id = event.rec_id
# Check if into new pool
if chk_rec_id > self.pool_rec_id:
if len(self.rec_ids) != 0:
self.msg_target.warning('Not all priming events were available. Missing: {0}'.format(str(self.rec_ids)))
self._start_pool()
raise CheckpointRecoveryComplete(self.pool_rec_id, '{0}'.format(self.name))
# Check if one of the priming events
if chk_rec_id in self.rec_ids:
self.prime_incidents.append(event)
self.rec_ids.remove(chk_rec_id)
return False
def set_checkpoint_from_pool(self):
''' Set the checkpoint based on the passed pool
Should only be called once the pool has been closed
'''
# Get the current pool from the ruleset
pool = self.get_pool()
# Gather information from pool
if pool.last_incident is None:
self.msg_target.debug('Not updated checkpoint because nothing processed (last_incident is None)')
return
t_min, t_data = self._gen_min_and_data(pool)
self.set_checkpoint(t_min, t_data)
return
def _start_pool(self):
''' Initialize and start the pool using the checkpoint data '''
# Get the current pool from the ruleset
pool = self.get_pool()
# Can't checkpoint unless pool is closed
if pool.state != POOL_STATE_NEW:
self.msg_target.warning('Unable start from checkpoint with pool that is not NEW')
return
if self.data is not None:
t_list = json.loads(self.data)
parse_data = re.compile(PARSE_DATA_DEF)
# Prime
t_dict = {}
for t_ent in t_list[2]:
t_p = parse_data.search(t_ent)
# Get extension time
if t_p.group('ext') is None:
t_ext = 0
else:
t_ext = long(t_p.group('ext'))
t_dict[long(t_p.group('rec_id'))] = (t_p.group('sup'), long(t_p.group('left')), t_ext)
# Know there will be one for each of these because they were used to create it
for t_event in self.rec_ids:
t_sup, t_min, t_dur = t_dict[t_event.rec_id]
pool.moved_forward.append(t_event)
if t_sup == 'S':
pool.suppressed[t_event.event_id].append(t_event)
else:
pool.incidents[t_event.event_id].append(t_event)
pool.duration = min(pool.duration + t_dur, pool.max_duration)
pool.min_time_incidents[t_event] = (t_min, 0, t_dur)
# Going to start pool so don't have to start when next incident added
pool._add_incident = pool._add_incident_ACTIVE_SUBSEQUENT
# Now start the pool with the checkpoint start time
pool.start(datetime.strptime(t_list[0], '%Y-%m-%d %H:%M:%S.%f'))
return
def _get_rec_ids_from_data(self):
''' Read from the data '''
if self.data is None:
return []
parse_data = re.compile(PARSE_DATA_DEF)
return [long(parse_data.search(t_list).group('rec_id')) for t_list in json.loads(self.data)[2]]
def _gen_min_and_data(self, pool):
''' generate the data and the min '''
mf_list = []
min_rec_id = pool.last_incident.rec_id
for incident in pool.min_time_incidents.keys():
mf_entry = str(incident.rec_id)
# Note that generated rec_id always starts with 1, so OK to subtract 1
min_rec_id = min(min_rec_id, incident.rec_id - 1)
tid = incident.get_incident_id()
if tid in pool.suppressed.keys() and incident in pool.suppressed[tid]:
mf_entry += 'S'
else:
mf_entry += 'U'
# For events moving forward: update duration, add to moving forward list
min_left, added_at_sec, dur_ext = pool.min_time_incidents[incident]
# Calculate new min time in pool (note: we know positive because if it wasn't, it
# would not have moved forward -- cleanup in close)
mf_entry += str(min_left - (pool.duration - added_at_sec))
if dur_ext != 0:
mf_entry += '+{0}'.format(str(dur_ext))
mf_list.append('"{0}"'.format(mf_entry))
if pool.start_time is None:
return (None, None)
else:
t_time = pool.start_time + timedelta(seconds=pool.duration)
return (min_rec_id, '["{0}", "{1}", [{2}]]'.format(str(t_time), pool.last_incident.rec_id, ','.join(mf_list)))
