def runworkers(): """Starts the re:dash query executors/workers.""" def stop_handler(signum, frame): logging.warning("Exiting; waiting for workers") data_manager.stop_workers() exit() signal.signal(signal.SIGTERM, stop_handler) signal.signal(signal.SIGINT, stop_handler) old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT) logging.info("Workers started.") while True: try: data_manager.refresh_queries() data_manager.report_status() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60)
def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error( "Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
redis_connection.delete(*query_hashes) starting_query_results_count = models.QueryResult.select().count() jobs_count = 5000 workers_count = 10 print "Creating jobs..." for i in xrange(jobs_count): query = "SELECT {}".format(i) print "Inserting: {}".format(query) data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count( ) - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False data_manager.stop_workers() qr_count = 0
if query_hashes: redis_connection.delete(*query_hashes) starting_query_results_count = models.QueryResult.select().count() jobs_count = 5000 workers_count = 10 print "Creating jobs..." for i in xrange(jobs_count): query = "SELECT {}".format(i) print "Inserting: {}".format(query) data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count() - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False data_manager.stop_workers() qr_count = 0 for qr in models.QueryResult.select():