def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error( "Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
def stop_handler(signum, frame): logging.warning("Exiting; waiting for workers") data_manager.stop_workers() exit()
priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count( ) - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False data_manager.stop_workers() qr_count = 0 for qr in models.QueryResult.select(): number = int(qr.query.split()[1]) data_number = json.loads(qr.data)['rows'][0].values()[0] if number != data_number: print "Oops? {} != {} ({})".format(number, data_number, qr.id) qr_count += 1 print "Verified {} query results.".format(qr_count) print "Done."
print "Inserting: {}".format(query) data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count() - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False data_manager.stop_workers() qr_count = 0 for qr in models.QueryResult.select(): number = int(qr.query.split()[1]) data_number = json.loads(qr.data)['rows'][0].values()[0] if number != data_number: print "Oops? {} != {} ({})".format(number, data_number, qr.id) qr_count += 1 print "Verified {} query results.".format(qr_count) print "Done."