def __dbm__(self):
def default_handler(target, event):
if isinstance(getattr(event, 'payload', None), Exception):
raise event.payload
log.warning('unsupported event ignored: %s' % type(event))
def check_sources_started(self, _locals, target, event):
_locals['countdown'] -= 1
if _locals['countdown'] == 0:
self._dbm_ready.set()
_locals = {'countdown': len(self._nl)}
# init the events map
event_map = {cmsg_event: [lambda t, x: x.payload.set()],
cmsg_failed: [lambda t, x: (self
.schema
.mark(t, 1))],
cmsg_sstart: [partial(check_sources_started,
self, _locals)]}
self._event_map = event_map
event_queue = self._event_queue
if self._db_provider == 'sqlite3':
self._db = sqlite3.connect(self._db_spec)
elif self._db_provider == 'psycopg2':
self._db = psycopg2.connect(**self._db_spec)
self.schema = schema.init(self,
self._db,
self._db_provider,
self._db_rtnl_log,
id(threading.current_thread()))
for spec in self._nl:
spec['event'] = None
self.sources.add(**spec)
for (event, handlers) in self.schema.event_map.items():
for handler in handlers:
self.register_handler(event, handler)
stop = False
reschedule = []
while not stop:
events = Events(event_queue.get(), reschedule)
reschedule = []
try:
for event in events:
handlers = event_map.get(event.__class__,
[default_handler, ])
for handler in tuple(handlers):
try:
target = event['header']['target']
handler(target, event)
except RescheduleException:
if 'rcounter' not in event['header']:
event['header']['rcounter'] = 0
if event['header']['rcounter'] < 3:
event['header']['rcounter'] += 1
self.log.debug('reschedule %s' % (event, ))
reschedule.append(event)
else:
self.log.error('drop %s' % (event, ))
except InvalidateHandlerException:
try:
handlers.remove(handler)
except:
self.log.error('could not invalidate '
'event handler:\n%s'
% traceback.format_exc())
except ShutdownException:
stop = True
break
except DBMExitException:
return
except:
self.log.error('could not load event:\n%s\n%s'
% (event, traceback.format_exc()))
if time.time() - self.gctime > config.gc_timeout:
self.gctime = time.time()
except Exception as e:
self.log.error('exception <%s> in source %s' % (e, target))
# restart the target
try:
self.sources[target].restart(reason=e)
except KeyError:
pass
# release all the sources
for target in tuple(self.sources.cache):
source = self.sources.remove(target, sync=False)
if source is not None and source.th is not None:
source.shutdown.set()
source.th.join()
self.log.debug('flush DB for the target %s' % target)
self.schema.flush(target)
# close the database
self.schema.commit()
self.schema.close()
def __dbm__(self):
def default_handler(target, event):
if isinstance(event, Exception):
raise event
log.warning('unsupported event ignored: %s' % type(event))
def check_sources_started(self, _locals, target, event):
_locals['countdown'] -= 1
if _locals['countdown'] == 0:
self._dbm_ready.set()
_locals = {'countdown': len(self._nl)}
# init the events map
event_map = {type(self._dbm_ready): [lambda t, x: x.set()],
MarkFailed: [lambda t, x: (self
.schema
.mark(t, 1))],
SyncStart: [partial(check_sources_started,
self, _locals)]}
self._event_map = event_map
event_queue = self._event_queue
self.__initdb__()
self.schema = schema.init(self,
self._db,
self._db_provider,
self._db_rtnl_log,
id(threading.current_thread()))
for spec in self._nl:
spec['event'] = None
self.sources.add(**spec)
for (event, handlers) in self.schema.event_map.items():
for handler in handlers:
self.register_handler(event, handler)
while True:
target, events = event_queue.get()
try:
if events is None:
continue
# if nlm_generator is True, an exception can come
# here while iterating events
for event in events:
handlers = event_map.get(event.__class__,
[default_handler, ])
for handler in tuple(handlers):
try:
handler(target, event)
except InvalidateHandlerException:
try:
handlers.remove(handler)
except:
log.error('could not invalidate '
'event handler:\n%s'
% traceback.format_exc())
except ShutdownException:
for target, source in self.sources.cache.items():
source.shutdown.set()
except DBMExitException:
return
except:
log.error('could not load event:\n%s\n%s'
% (event, traceback.format_exc()))
if time.time() - self.gctime > config.gc_timeout:
self.gctime = time.time()
except Exception as e:
self.log.error('exception <%s> in source %s' % (e, target))
# restart the target
self.sources[target].restart(reason=e)