def runworkers():
"""Starts the re:dash query executors/workers."""
def stop_handler(signum, frame):
logging.warning("Exiting; waiting for workers")
data_manager.stop_workers()
exit()
signal.signal(signal.SIGTERM, stop_handler)
signal.signal(signal.SIGINT, stop_handler)
old_workers = data_manager.redis_connection.smembers('workers')
data_manager.redis_connection.delete('workers')
logging.info("Cleaning old workers: %s", old_workers)
data_manager.start_workers(settings.WORKERS_COUNT)
logging.info("Workers started.")
while True:
try:
data_manager.refresh_queries()
data_manager.report_status()
except Exception as e:
logging.error("Something went wrong with refreshing queries...")
logging.exception(e)
time.sleep(60)
def runworkers():
"""Starts the re:dash query executors/workers."""
try:
old_workers = data_manager.redis_connection.smembers('workers')
data_manager.redis_connection.delete('workers')
logging.info("Cleaning old workers: %s", old_workers)
data_manager.start_workers(settings.WORKERS_COUNT,
settings.CONNECTION_ADAPTER,
settings.CONNECTION_STRING)
logging.info("Workers started.")
while True:
try:
data_manager.refresh_queries()
except Exception as e:
logging.error(
"Something went wrong with refreshing queries...")
logging.exception(e)
time.sleep(60)
except KeyboardInterrupt:
logging.warning("Exiting; waiting for threads")
data_manager.stop_workers()
def runworkers():
"""Starts the re:dash query executors/workers."""
def stop_handler(signum, frame):
logging.warning("Exiting; waiting for workers")
data_manager.stop_workers()
exit()
signal.signal(signal.SIGTERM, stop_handler)
signal.signal(signal.SIGINT, stop_handler)
old_workers = data_manager.redis_connection.smembers('workers')
data_manager.redis_connection.delete('workers')
logging.info("Cleaning old workers: %s", old_workers)
data_manager.start_workers(settings.WORKERS_COUNT)
logging.info("Workers started.")
while True:
try:
data_manager.refresh_queries()
data_manager.report_status()
except Exception as e:
logging.error("Something went wrong with refreshing queries...")
logging.exception(e)
time.sleep(60)
def test_skips_queries_with_no_ttl(self):
query = query_factory.create(ttl=-1)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
query_hash=query.query_hash)
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
self.assertFalse(add_job_mock.called)
def test_enqueues_outdated_queries(self):
query = query_factory.create(ttl=60)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
query_hash=query.query_hash)
query.latest_query_data = query_result
query.save()
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
add_job_mock.assert_called_with(query.query, worker.Job.LOW_PRIORITY, query.data_source)
def test_skips_queries_with_no_ttl(self):
query = query_factory.create(ttl=-1)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(
minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at,
query=query.query,
query_hash=query.query_hash)
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
self.assertFalse(add_job_mock.called)
def test_enqueues_query_with_correct_data_source(self):
query = query_factory.create(ttl=60)
query2 = query_factory.create(ttl=60, query=query.query, query_hash=query.query_hash)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at, query=query.query,
query_hash=query.query_hash)
query.latest_query_data = query_result
query2.latest_query_data = query_result
query.save()
query2.save()
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
add_job_mock.assert_has_calls([call(query2.query, worker.Job.LOW_PRIORITY, query2.data_source), call(query.query, worker.Job.LOW_PRIORITY, query.data_source)], any_order=True)
self.assertEquals(2, add_job_mock.call_count)
def test_enqueues_outdated_queries(self):
query = query_factory.create(ttl=60)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(
minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at,
query=query.query,
query_hash=query.query_hash)
query.latest_query_data = query_result
query.save()
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
add_job_mock.assert_called_with(query.query,
worker.Job.LOW_PRIORITY,
query.data_source)
def runworkers():
"""Starts the re:dash query executors/workers."""
try:
old_workers = data_manager.redis_connection.smembers('workers')
data_manager.redis_connection.delete('workers')
logging.info("Cleaning old workers: %s", old_workers)
data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
logging.info("Workers started.")
while True:
try:
data_manager.refresh_queries()
except Exception as e:
logging.error("Something went wrong with refreshing queries...")
logging.exception(e)
time.sleep(60)
except KeyboardInterrupt:
logging.warning("Exiting; waiting for threads")
data_manager.stop_workers()
def test_enqueues_query_with_correct_data_source(self):
query = query_factory.create(ttl=60)
query2 = query_factory.create(ttl=60,
query=query.query,
query_hash=query.query_hash)
retrieved_at = datetime.datetime.utcnow() - datetime.timedelta(
minutes=10)
query_result = query_result_factory.create(retrieved_at=retrieved_at,
query=query.query,
query_hash=query.query_hash)
query.latest_query_data = query_result
query2.latest_query_data = query_result
query.save()
query2.save()
with patch('redash.data.Manager.add_job') as add_job_mock:
data_manager.refresh_queries()
add_job_mock.assert_has_calls([
call(query2.query, worker.Job.LOW_PRIORITY,
query2.data_source),
call(query.query, worker.Job.LOW_PRIORITY, query.data_source)
],
any_order=True)
self.assertEquals(2, add_job_mock.call_count)
