def runworkers(): """Starts the re:dash query executors/workers.""" def stop_handler(signum, frame): logging.warning("Exiting; waiting for workers") data_manager.stop_workers() exit() signal.signal(signal.SIGTERM, stop_handler) signal.signal(signal.SIGINT, stop_handler) old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT) logging.info("Workers started.") while True: try: data_manager.refresh_queries() data_manager.report_status() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60)
def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error( "Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
def test_skips_queries_with_no_ttl(self): query = query_factory.create(ttl=-1) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() self.assertFalse(add_job_mock.called)
def test_enqueues_outdated_queries(self): query = query_factory.create(ttl=60) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) query.latest_query_data = query_result query.save() with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
def test_skips_queries_with_no_ttl(self): query = query_factory.create(ttl=-1) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta( minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() self.assertFalse(add_job_mock.called)
def test_enqueues_query_with_correct_data_source(self): query = query_factory.create(ttl=60) query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) query.latest_query_data = query_result query2.latest_query_data = query_result query.save() query2.save() with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True) self.assertEquals(2, add_job_mock.call_count)
def test_enqueues_outdated_queries(self): query = query_factory.create(ttl=60) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta( minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) query.latest_query_data = query_result query.save() with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
def runworkers(): """Starts the re:dash query executors/workers.""" try: old_workers = data_manager.redis_connection.smembers('workers') data_manager.redis_connection.delete('workers') logging.info("Cleaning old workers: %s", old_workers) data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING) logging.info("Workers started.") while True: try: data_manager.refresh_queries() except Exception as e: logging.error("Something went wrong with refreshing queries...") logging.exception(e) time.sleep(60) except KeyboardInterrupt: logging.warning("Exiting; waiting for threads") data_manager.stop_workers()
def test_enqueues_query_with_correct_data_source(self): query = query_factory.create(ttl=60) query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash) retrieved_at = datetime.datetime.utcnow() - datetime.timedelta( minutes=10) query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query, query_hash=query.query_hash) query.latest_query_data = query_result query2.latest_query_data = query_result query.save() query2.save() with patch('redash.data.Manager.add_job') as add_job_mock: data_manager.refresh_queries() add_job_mock.assert_has_calls([ call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source) ], any_order=True) self.assertEquals(2, add_job_mock.call_count)