def on_quit(self): #pragma no cover
self.event_publisher.close()
if self.subscriber_thread:
self.stop_listener()
for stream, coverage in self._coverages.iteritems():
try:
coverage.close(timeout=5)
except:
log.exception('Problems closing the coverage')
self._coverages.clear()
TransformStreamListener.on_quit(self)
BaseIngestionWorker.on_quit(self)
def on_quit(self): #pragma no cover
self.event_publisher.close()
if self.subscriber_thread:
self.stop_listener()
for stream, coverage in self._coverages.iteritems():
try:
coverage.close(timeout=5)
except:
log.exception('Problems closing the coverage')
self._coverages.clear()
TransformStreamListener.on_quit(self)
BaseIngestionWorker.on_quit(self)
def on_start(self): #pragma no cover
log.info('Starting Demuxer')
TransformStreamListener.on_start(self)
log.info('----------')
self.output_streams = self.CFG.get_safe('process.out_streams')
if self.output_streams is None or not isinstance(self.output_streams, list):
log.error('(%s) Failed to demux, I/O configuration is incorrect: (%s)', self.id, self.output_streams)
return
self.publishers = []
for stream in self.output_streams:
log.info(" -> %s", stream)
self.publishers.append(StreamPublisher(process=self, stream_id=stream))
def __init__(self, *args,**kwargs):
TransformStreamListener.__init__(self, *args, **kwargs)
BaseIngestionWorker.__init__(self, *args, **kwargs)
#--------------------------------------------------------------------------------
# Ingestion Cache
# - Datasets
# - Coverage instances
#--------------------------------------------------------------------------------
self._datasets = collections.OrderedDict()
self._coverages = collections.OrderedDict()
self._bad_coverages = {}
self.time_stats = Accumulator(format='%3f')
# unique ID to identify this worker in log msgs
self._id = uuid.uuid1()
def __init__(self, *args, **kwargs):
TransformStreamListener.__init__(self, *args, **kwargs)
BaseIngestionWorker.__init__(self, *args, **kwargs)
#--------------------------------------------------------------------------------
# Ingestion Cache
# - Datasets
# - Coverage instances
#--------------------------------------------------------------------------------
self._datasets = collections.OrderedDict()
self._coverages = collections.OrderedDict()
self._bad_coverages = {}
self.time_stats = Accumulator(format='%3f')
# unique ID to identify this worker in log msgs
self._id = uuid.uuid1()
def on_start(self): #pragma no cover
log.info('Starting Demuxer')
TransformStreamListener.on_start(self)
log.info('----------')
self.output_streams = self.CFG.get_safe('process.out_streams')
if self.output_streams is None or not isinstance(
self.output_streams, list):
log.error(
'(%s) Failed to demux, I/O configuration is incorrect: (%s)',
self.id, self.output_streams)
return
self.publishers = []
for stream in self.output_streams:
log.info(" -> %s", stream)
self.publishers.append(
StreamPublisher(process=self, stream_id=stream))
def on_start(self):
TransformStreamListener.on_start(self)
self.document_key = self.CFG.get_safe('process.document_key')
self.stored_value_manager = StoredValueManager(self.container)
def on_start(self):
TransformStreamListener.on_start(self)
self.document_key = self.CFG.get_safe('process.document_key')
self.stored_value_manager = StoredValueManager(self.container)
def on_quit(self): #pragma no cover
TransformStreamListener.on_quit(self)
for publisher in self.publishers:
publisher.close()
def on_quit(self): #pragma no cover
TransformStreamListener.on_quit(self)
for publisher in self.publishers:
publisher.close()
