def on_quit(self): #pragma no cover self.event_publisher.close() if self.subscriber_thread: self.stop_listener() for stream, coverage in self._coverages.iteritems(): try: coverage.close(timeout=5) except: log.exception('Problems closing the coverage') self._coverages.clear() TransformStreamListener.on_quit(self) BaseIngestionWorker.on_quit(self)
def on_start(self): #pragma no cover log.info('Starting Demuxer') TransformStreamListener.on_start(self) log.info('----------') self.output_streams = self.CFG.get_safe('process.out_streams') if self.output_streams is None or not isinstance(self.output_streams, list): log.error('(%s) Failed to demux, I/O configuration is incorrect: (%s)', self.id, self.output_streams) return self.publishers = [] for stream in self.output_streams: log.info(" -> %s", stream) self.publishers.append(StreamPublisher(process=self, stream_id=stream))
def __init__(self, *args,**kwargs): TransformStreamListener.__init__(self, *args, **kwargs) BaseIngestionWorker.__init__(self, *args, **kwargs) #-------------------------------------------------------------------------------- # Ingestion Cache # - Datasets # - Coverage instances #-------------------------------------------------------------------------------- self._datasets = collections.OrderedDict() self._coverages = collections.OrderedDict() self._bad_coverages = {} self.time_stats = Accumulator(format='%3f') # unique ID to identify this worker in log msgs self._id = uuid.uuid1()
def __init__(self, *args, **kwargs): TransformStreamListener.__init__(self, *args, **kwargs) BaseIngestionWorker.__init__(self, *args, **kwargs) #-------------------------------------------------------------------------------- # Ingestion Cache # - Datasets # - Coverage instances #-------------------------------------------------------------------------------- self._datasets = collections.OrderedDict() self._coverages = collections.OrderedDict() self._bad_coverages = {} self.time_stats = Accumulator(format='%3f') # unique ID to identify this worker in log msgs self._id = uuid.uuid1()
def on_start(self): #pragma no cover log.info('Starting Demuxer') TransformStreamListener.on_start(self) log.info('----------') self.output_streams = self.CFG.get_safe('process.out_streams') if self.output_streams is None or not isinstance( self.output_streams, list): log.error( '(%s) Failed to demux, I/O configuration is incorrect: (%s)', self.id, self.output_streams) return self.publishers = [] for stream in self.output_streams: log.info(" -> %s", stream) self.publishers.append( StreamPublisher(process=self, stream_id=stream))
def on_start(self): TransformStreamListener.on_start(self) self.document_key = self.CFG.get_safe('process.document_key') self.stored_value_manager = StoredValueManager(self.container)
def on_quit(self): #pragma no cover TransformStreamListener.on_quit(self) for publisher in self.publishers: publisher.close()