def run(self): # TODO set up exception handlers # TODO implement global shutdown function (child to parent) # TODO global shutdown on unhandled exception # TODO rename pair to symbol LOG.info("Aggregator running on PID %d", os.getpid()) # Setup uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() self.config = DynamicConfig(loop=loop, file_name=self.config_file) loop.create_task(self.loop(loop=loop)) self.event = threading.Event() # sleep control for write threads setup_event_loop_signal_handlers(loop, self._stop_on_signal) try: loop.run_forever() except KeyboardInterrupt: pass except Exception: LOG.error("Aggregator running on PID %d died due to exception", os.getpid(), exc_info=True)
def run(self): LOG.info("Aggregator running on PID %d", os.getpid()) loop = asyncio.get_event_loop() self.config = DynamicConfig(loop=loop, file_name=self.config_file) loop.create_task(self.loop(loop=loop)) self.event = threading.Event() # sleep control for write threads setup_event_loop_signal_handlers(loop, self._stop_on_signal) try: loop.run_forever() except KeyboardInterrupt: pass except Exception: LOG.error("Aggregator running on PID %d died due to exception", os.getpid(), exc_info=True)
def run(self): LOG.info("Aggregator running on PID %d", os.getpid()) loop = asyncio.get_event_loop() self.config = DynamicConfig(file_name=self.config_file) loop.create_task(self.loop()) try: loop.run_forever() except KeyboardInterrupt: pass
def run(self): LOG.info("Starting cryptostore") LOG.info("Cryptostore running on PID %d", os.getpid()) self.spawner.start() LOG.info("Spawner started") self.aggregator = Aggregator(config_file=self.cfg_path) self.aggregator.start() LOG.info("Aggregator started") loop = asyncio.get_event_loop() self.config = DynamicConfig(file_name=self.cfg_path, callback=self._reconfigure) LOG.info("Cryptostore started") loop.run_forever()
def run(self): LOG.info("Starting cryptostore") LOG.info("Cryptostore running on PID %d", os.getpid()) self.spawner.start() LOG.info("Spawner started") self.aggregator.start() LOG.info("Aggregator started") loop = asyncio.get_event_loop() setup_event_loop_signal_handlers(loop, self._stop_on_signal) self.config = DynamicConfig(loop=loop, file_name=self.cfg_path, callback=self._reconfigure) LOG.info("Cryptostore started") loop.run_forever() LOG.info("Cryptostore main process stopped")
def run(self): LOG.info("Aggregator running on PID %d", os.getpid()) loop = asyncio.get_event_loop() self.config = DynamicConfig(file_name=self.config_file) self.mysql_client = MySQLClient(loop) _last_arb_id = loop.run_until_complete( self.mysql_client.get_last_arb_id()) if not _last_arb_id == None: self.last_arb_id = int(_last_arb_id[0]) else: self.last_arb_id = 0 loop.create_task(self.loop()) try: loop.run_forever() except KeyboardInterrupt: pass except Exception: LOG.error("Aggregator running on PID %d died due to exception", os.getpid(), exc_info=True)
def run(self): LOG.info("Starting cryptostore") LOG.info("Cryptostore running on PID %d", os.getpid()) # Setup uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() def exception_handler(loop, context): # TODO figure our exception handling and interrupts for all the processes # TODO https://quantlane.com/blog/ensure-asyncio-task-exceptions-get-logged/ # TODO make an untility funcrion and move to utils? # first, handle with default handler loop.default_exception_handler(context) signal.raise_signal(signal.SIGTERM) loop.set_exception_handler(exception_handler) setup_event_loop_signal_handlers(loop, self._stop_on_signal) self.config = DynamicConfig(loop=loop, file_name=self.cfg_path, callback=self._reconfigure) start_prometheus(self.config.config) self.spawner.start() LOG.info("Spawner started") self.aggregator.start() LOG.info("Aggregator started") LOG.info("Cryptostore started") try: loop.run_forever() except Exception as e: LOG.info("Loop crashed") LOG.info("Cryptostore main process stopped")
class Aggregator(Process): def __init__(self, config_file=None): self.config_file = config_file super().__init__() self.daemon = True self.terminating = False def run(self): LOG.info("Aggregator running on PID %d", os.getpid()) loop = asyncio.get_event_loop() self.config = DynamicConfig(loop=loop, file_name=self.config_file) loop.create_task(self.loop(loop=loop)) self.event = threading.Event() # sleep control for write threads setup_event_loop_signal_handlers(loop, self._stop_on_signal) try: loop.run_forever() except KeyboardInterrupt: pass except Exception: LOG.error("Aggregator running on PID %d died due to exception", os.getpid(), exc_info=True) def _stop_on_signal(self, sig, loop): if self.terminating: LOG.info("Aggregator on PID %d is already being stopped...", os.getpid()) return LOG.info("Stopping Aggregator on %d due to signal %d", os.getpid(), sig) self.terminating = True self.config.set_terminating() self.event.set() if 'write_on_stop' in self.config and self.config.write_on_stop \ and 'exchanges' in self.config and self.config.exchanges: stop_event_loop(loop, self._write_storage(loop=loop, write_on_stop=True)) else: stop_event_loop(loop) async def loop(self, loop): if self.config.cache == 'redis': self.cache = Redis(ip=self.config.redis['ip'], port=self.config.redis['port'], password=os.environ.get('REDIS_PASSWORD', None) or self.config.redis.get('password', None), socket=self.config.redis.socket, del_after_read=self.config.redis['del_after_read'], flush=self.config.redis['start_flush'], retention=self.config.redis.retention_time if 'retention_time' in self.config.redis else None) elif self.config.cache == 'kafka': self.cache = Kafka(self.config.kafka['ip'], self.config.kafka['port'], flush=self.config.kafka['start_flush']) interval = self.config.storage_interval time_partition = False multiplier = 1 if not isinstance(interval, int): if len(interval) > 1: multiplier = int(interval[:-1]) interval = interval[-1] base_interval = interval if interval in {'M', 'H', 'D'}: time_partition = True if interval == 'M': interval = 60 * multiplier elif interval == 'H': interval = 3600 * multiplier else: interval = 86400 * multiplier self.parquet_buffer = dict() while not self.terminating: start, end = None, None try: aggregation_start = time.time() if time_partition: interval_start = aggregation_start if end: interval_start = end + timedelta(seconds=interval + 1) start, end = get_time_interval(interval_start, base_interval, multiplier=multiplier) if 'exchanges' in self.config and self.config.exchanges: await self._write_storage(loop=loop, start=start, end=end) total = time.time() - aggregation_start wait = interval - total if wait <= 0: LOG.warning("Storage operations currently take %.1f seconds, longer than the interval of %d", total, interval) wait = 0.5 try: await asyncio.sleep(delay=wait, loop=loop) except asyncio.CancelledError as e: pass else: try: await asyncio.sleep(delay=30, loop=loop) except asyncio.CancelledError as e: pass except Exception: LOG.error("Aggregator running on PID %d died due to exception", os.getpid(), exc_info=True) raise LOG.info("Aggregator running on PID %d stopped", os.getpid()) async def _write_storage(self, loop, start=None, end=None, write_on_stop=False): if write_on_stop: LOG.info("Writing cached data before stopping...") else: LOG.info("Writing cached data...") max_workers = self.config.num_write_threads if 'num_write_threads' in self.config else 1 with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool: futures = [] for exchange in self.config.exchanges: for dtype in self.config.exchanges[exchange]: # Skip over the retries arg in the config if present. if dtype in {'retries', 'channel_timeouts', 'http_proxy'}: continue for pair in self.config.exchanges[exchange][dtype] if 'symbols' not in \ self.config.exchanges[exchange][dtype] else \ self.config.exchanges[exchange][dtype]['symbols']: futures.append(loop.run_in_executor(pool, self._write_pair_blocking, exchange, dtype, pair, start, end, write_on_stop)) await asyncio.gather(*futures, loop=loop, return_exceptions=False) LOG.info("Write finished") def _write_pair_blocking(self, exchange, dtype, pair, start, end, write_on_stop): LOG.info('Reading cache for %s-%s-%s', exchange, dtype, pair) store = Storage(self.config, parquet_buffer=self.parquet_buffer) data = self.cache.read(exchange, dtype, pair, start=start, end=end) if len(data) == 0: LOG.info('No data for %s-%s-%s', exchange, dtype, pair) return store.aggregate(data) retries = 0 while ((not self.terminating) or write_on_stop): if retries > self.config.storage_retries: LOG.error("Failed to write after %d retries", self.config.storage_retries) raise EngineWriteError try: # retrying this is ok, provided every # engine clears its internal buffer after writing successfully. LOG.info('Writing cached data to store for %s-%s-%s', exchange, dtype, pair) store.write(exchange, dtype, pair, time.time()) except OSError as e: LOG.warning('Could not write %s-%s-%s. %s', exchange, dtype, pair, e) if write_on_stop: break if e.errno == 112: # Host is down retries += 1 self.event.wait(self.config.storage_retry_wait) else: raise except EngineWriteError: LOG.warning('Could not write %s-%s-%s. %s', exchange, dtype, pair, e) if write_on_stop: break retries += 1 self.event.wait(self.config.storage_retry_wait) else: break LOG.info('Deleting cached data for %s-%s-%s', exchange, dtype, pair) self.cache.delete(exchange, dtype, pair) LOG.info('Write Complete %s-%s-%s', exchange, dtype, pair)