class PollingAutoUpdater(object):
def __init__(self, resolved_paths, reloader, interval):
self._paths = resolved_paths
self._reloader = reloader
self._file_times = self._check_file_times()
self._timer = RepeatingTimer(interval, self._poll)
self._timer.start()
def stop(self):
self._timer.stop()
def _poll(self):
new_times = self._check_file_times()
changed = False
for file_path, file_time in six.iteritems(self._file_times):
if new_times.get(file_path) is not None and new_times.get(
file_path) != file_time:
changed = True
break
self._file_times = new_times
if changed:
self._reloader()
def _check_file_times(self):
ret = {}
for path in self._paths:
try:
ret[path] = os.path.getmtime(path)
except:
ret[path] = None
return ret
class DefaultEventProcessor(EventProcessor):
def __init__(self, config, http=None):
self._queue = queue.Queue(config.events_max_pending)
self._flush_timer = RepeatingTimer(config.flush_interval, self.flush)
self._users_flush_timer = RepeatingTimer(
config.user_keys_flush_interval, self._flush_users)
self._flush_timer.start()
self._users_flush_timer.start()
self._close_lock = Lock()
self._closed = False
EventDispatcher(self._queue, config, http)
def send_event(self, event):
event['creationDate'] = int(time.time() * 1000)
self._queue.put(EventProcessorMessage('event', event))
def flush(self):
self._queue.put(EventProcessorMessage('flush', None))
def stop(self):
with self._close_lock:
if self._closed:
return
self._closed = True
self._flush_timer.stop()
self._users_flush_timer.stop()
self.flush()
self._post_message_and_wait('stop')
def _flush_users(self):
self._queue.put(EventProcessorMessage('flush_users', None))
# Used only in tests
def _wait_until_inactive(self):
self._post_message_and_wait('test_sync')
def _post_message_and_wait(self, type):
reply = Event()
self._queue.put(EventProcessorMessage(type, reply))
reply.wait()
class DefaultEventProcessor(EventProcessor):
def __init__(self,
config,
http=None,
dispatcher_class=None,
diagnostic_accumulator=None):
self._inbox = queue.Queue(config.events_max_pending)
self._inbox_full = False
self._flush_timer = RepeatingTimer(config.flush_interval, self.flush)
self._users_flush_timer = RepeatingTimer(
config.user_keys_flush_interval, self._flush_users)
self._flush_timer.start()
self._users_flush_timer.start()
if diagnostic_accumulator is not None:
self._diagnostic_event_timer = RepeatingTimer(
config.diagnostic_recording_interval, self._send_diagnostic)
self._diagnostic_event_timer.start()
else:
self._diagnostic_event_timer = None
self._close_lock = Lock()
self._closed = False
(dispatcher_class or EventDispatcher)(self._inbox, config, http,
diagnostic_accumulator)
def send_event(self, event):
event['creationDate'] = int(time.time() * 1000)
self._post_to_inbox(EventProcessorMessage('event', event))
def flush(self):
self._post_to_inbox(EventProcessorMessage('flush', None))
def stop(self):
with self._close_lock:
if self._closed:
return
self._closed = True
self._flush_timer.stop()
self._users_flush_timer.stop()
if self._diagnostic_event_timer:
self._diagnostic_event_timer.stop()
self.flush()
# Note that here we are not calling _post_to_inbox, because we *do* want to wait if the inbox
# is full; an orderly shutdown can't happen unless these messages are received.
self._post_message_and_wait('stop')
def _post_to_inbox(self, message):
try:
self._inbox.put(message, block=False)
except queue.Full:
if not self._inbox_full:
# possible race condition here, but it's of no real consequence - we'd just get an extra log line
self._inbox_full = True
log.warning(
"Events are being produced faster than they can be processed; some events will be dropped"
)
def _flush_users(self):
self._inbox.put(EventProcessorMessage('flush_users', None))
def _send_diagnostic(self):
self._inbox.put(EventProcessorMessage('diagnostic', None))
# Used only in tests
def _wait_until_inactive(self):
self._post_message_and_wait('test_sync')
def _post_message_and_wait(self, type):
reply = Event()
self._inbox.put(EventProcessorMessage(type, reply))
reply.wait()
# These magic methods allow use of the "with" block in tests
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.stop()