class ClientWorker(object):
"""test..."""
def __init__(
self, collection_name, terms, event, pipe, total, config_data):
self.log = logging.getLogger(self.__class__.__name__)
self.collection_name = collection_name
self.event = event
self.terms = terms
self.raw_pipe = pipe
self.config_data = config_data
self.listener = None
self.client = None
self.config = ConfigFile(config_data=self.config_data)
self.initial_total = total if total is not None else 0
self.pipe = PipeMessenger(self.raw_pipe)
self.client_status = CaptureStatus(CaptureStatus.STATUS_UNKNOWN)
def initialize(self):
"""initialize the worker."""
output_config = self.config.getValue("output", {})
self.listener = RotatingFileListener(
collection_name=self.collection_name,
**output_config
)
self.listener.data_callback = functools.partial(
self.dataCallback,
(self,)
)
self.listener.stats.total = self.initial_total
# create client
self.client = TwitterClient(
listener=self.listener,
keywords=self.terms
)
self.client.configure(self.config)
# last stat update
self.last_stat_update_time = datetime.now()
def updateStatus(self, status):
if status != self.client_status.value:
self.client_status.value = status
self.pipe.sendStatus(status)
def shutdown(self):
"""shutdown the worker."""
self.updateStatus(CaptureStatus.STATUS_STOPPING)
try:
self.client.shutdown()
except Exception:
self.log.exception("Exception during client shutdown")
try:
self.listener.shutdown()
except Exception:
self.log.exception("Exception while shutting down listener")
self.updateStatus(CaptureStatus.STATUS_STOPPED)
def run(self):
"""run the collection."""
self.updateStatus(CaptureStatus.STATUS_STARTING)
self.client.initialize()
self.client.run()
self.log.debug("run returned. ClientStatus -> Stopped")
# self.updateStatus(CaptureStatus.STATUS_STOPPED)
def dataCallback(self, listener, stats):
"""handle data callback by sending periodic updates."""
# self.log.debug("worker callback!")
# self.log.debug("self: %s", repr(self))
# self.log.debug("listener: %s", repr(listener))
# self.log.debug("stats: %s", repr(stats))
# quitting?
if self.event is None or self.event.is_set():
return False
# update status if necessary
self.updateStatus(CaptureStatus.STATUS_STARTED)
now = datetime.now()
delta = now - self.last_stat_update_time
if delta.total_seconds() > 5:
received = stats.received
stats.calculate_rate()
self.pipe.sendUpdate(now, received, stats.rate, stats.total)
self.last_stat_update_time = now
return True
