def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, None,
{
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query_text + '2', query.data_source, query.user_id,
None, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query_text + '3', query.data_source, query.user_id,
None, {
'Username': '******',
'Query ID': query.id
})
self.assertEqual(3, execute_query.apply_async.call_count)
self.assertEqual(3,
redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(
0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = MagicMock(side_effect=gen_hash)
enqueue_query(query.query, query.data_source, True, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query + '2', query.data_source, True, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query + '3', query.data_source, True, {'Username': '******', 'Query ID': query.id})
self.assertEqual(3, execute_query.apply_async.call_count)
self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def test_removes_oldest_items_first(self):
remove_count = QueryTaskTracker.prune(self.list, 50)
self.assertEqual(remove_count, 50)
self.assertEqual(redis_connection.zcard(self.list), 50)
self.assertEqual(redis_connection.zscore(self.list, 'k:99'), 99.0)
self.assertIsNone(redis_connection.zscore(self.list, 'k:1'))
def prune(cls, list_name, keep_count, max_keys=100):
count = redis_connection.zcard(list_name)
if count <= keep_count:
return 0
remove_count = min(max_keys, count - keep_count)
keys = redis_connection.zrange(list_name, 0, remove_count - 1)
redis_connection.delete(*keys)
redis_connection.zremrangebyrank(list_name, 0, remove_count - 1)
return remove_count
def prune(cls, list_name, keep_count, max_keys=100):
count = redis_connection.zcard(list_name)
if count <= keep_count:
return 0
remove_count = min(max_keys, count - keep_count)
keys = redis_connection.zrange(list_name, 0, remove_count - 1)
redis_connection.delete(*keys)
redis_connection.zremrangebyrank(list_name, 0, remove_count - 1)
return remove_count
def prune(cls, list_name, keep_count):
count = redis_connection.zcard(list_name)
if count <= keep_count:
return 0
remove_count = count - keep_count
keys = redis_connection.zrange(list_name, 0, remove_count - 1)
redis_connection.delete(keys)
redis_connection.zremrangebyrank(list_name, 0, remove_count - 1)
return remove_count
def test_multiple_enqueue_of_same_query(self):
query = self.factory.create_query()
execute_query.apply_async = MagicMock(side_effect=gen_hash)
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
self.assertEqual(1, execute_query.apply_async.call_count)
self.assertEqual(1,
redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(
0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def status_api():
status = {}
info = redis_connection.info()
status['redis_used_memory'] = info['used_memory_human']
status['queries_count'] = models.Query.select().count()
status['query_results_count'] = models.QueryResult.select().count()
status['dashboards_count'] = models.Dashboard.select().count()
status['widgets_count'] = models.Widget.select().count()
status['workers'] = [redis_connection.hgetall(w)
for w in redis_connection.smembers('workers')]
manager_status = redis_connection.hgetall('manager:status')
status['manager'] = manager_status
status['manager']['queue_size'] = redis_connection.zcard('jobs')
return jsonify(status)
def status_api():
status = {}
info = redis_connection.info()
status['redis_used_memory'] = info['used_memory_human']
status['queries_count'] = models.Query.select().count()
status['query_results_count'] = models.QueryResult.select().count()
status['dashboards_count'] = models.Dashboard.select().count()
status['widgets_count'] = models.Widget.select().count()
status['workers'] = [
redis_connection.hgetall(w)
for w in redis_connection.smembers('workers')
]
manager_status = redis_connection.hgetall('manager:status')
status['manager'] = manager_status
status['manager']['queue_size'] = redis_connection.zcard('jobs')
return jsonify(status)
def test_does_nothing_when_below_threshold(self):
remove_count = QueryTaskTracker.prune(self.list, 100)
self.assertEqual(remove_count, 0)
self.assertEqual(redis_connection.zcard(self.list), 100)
def test_does_nothing_when_below_threshold(self):
remove_count = QueryTaskTracker.prune(self.list, 100)
self.assertEqual(remove_count, 0)
self.assertEqual(redis_connection.zcard(self.list), 100)
