def main(channels, base_dir='.', qualities='source', first_hour=None, last_hour=None, metrics_port=8006, backdoor_port=0): """Segment coverage service""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] if first_hour is not None: first_hour = dateutil.parse(first_hour) if last_hour is not None: last_hour = dateutil.parse(last_hour) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) managers = [] workers = [] for channel in channels: logging.info( 'Starting coverage checks {} with {} as qualities in {}'.format( channel, ', '.join(qualities), base_dir)) manager = CoverageChecker(channel, qualities, base_dir, first_hour, last_hour) managers.append(manager) workers.append(gevent.spawn(manager.run)) def stop(): for manager in managers: manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() # Wait for any to die gevent.wait(workers, count=1) # If one has stopped, either: # 1. stop() was called and all are stopping # 2. one errored and we should stop all remaining and report the error # Our behaviour in both cases is the same: # 1. Tell all managers to gracefully stop stop() # 2. Wait (with timeout) until they've stopped gevent.wait(workers) # 3. Check if any of them failed. If they did, report it. If mulitple # failed, we report one arbitrarily. for worker in workers: worker.get() logging.info('Gracefully stopped')
def main(channels, base_dir='.', qualities='source', metrics_port=8002, static_nodes='', backdoor_port=0, start=None, delete_old=False, run_once=False, node_file=None, node_database=None, localhost=socket.gethostname(), download_concurrency=5, recent_cutoff=120): """Backfiller service.""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] static_nodes = static_nodes.split(',') if static_nodes else [] static_nodes = [static_node.strip() for static_node in static_nodes] if start is not None: try: start = float(start) logging.info('Backfilling last {} hours'.format(start)) except ValueError: start = dateutil.parse(start) logging.info('Backfilling since {}'.format(start)) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) logging.info('Starting backfilling {} with {} as qualities to {}'.format( ', '.join(channels), ', '.join(qualities), base_dir)) manager = BackfillerManager(base_dir, channels, qualities, static_nodes, start, delete_old, run_once, node_file, node_database, localhost, download_concurrency, recent_cutoff) def stop(): manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() manager.run() logging.info('Gracefully stopped')
def main(channels, base_dir='.', qualities='source', metrics_port=8002, static_nodes='', backdoor_port=0, start=None, delete_old=False, run_once=False, node_file=None, node_database=None, localhost=socket.gethostname(), download_concurrency=5, recent_cutoff=120): """Backfiller service.""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] static_nodes = static_nodes.split(',') if static_nodes else [] static_nodes = [static_node.strip() for static_node in static_nodes] if start is not None: try: start = float(start) logging.info('Backfilling last {} hours'.format(start)) except ValueError: start = dateutil.parse(start) logging.info('Backfilling since {}'.format(start)) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) managers = [] workers = [] for channel in channels: logging.info( 'Starting backfilling {} with {} as qualities to {}'.format( channel, ', '.join(qualities), base_dir)) manager = BackfillerManager(base_dir, channel, qualities, static_nodes, start, delete_old, run_once, node_file, node_database, localhost, download_concurrency, recent_cutoff) managers.append(manager) workers.append(gevent.spawn(manager.run)) def stop(): for manager in managers: manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() # Wait for any to die gevent.wait(workers, count=1) # If one has stopped, either: # 1. stop() was called and all are stopping # 2. one errored and we should stop all remaining and report the error # Our behaviour in both cases is the same: # 1. Tell all managers to gracefully stop stop() # 2. Wait (with timeout) until they've stopped gevent.wait(workers) # 3. Check if any of them failed. If they did, report it. If mulitple # failed, we report one arbitrarily. for worker in workers: worker.get() logging.info('Gracefully stopped')