def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query_text + '2', query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query_text + '3', query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) self.assertEqual(3, execute_query.apply_async.call_count) self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual( 0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = MagicMock(side_effect=gen_hash) enqueue_query(query.query, query.data_source, True, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query + '2', query.data_source, True, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query + '3', query.data_source, True, {'Username': '******', 'Query ID': query.id}) self.assertEqual(3, execute_query.apply_async.call_count) self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def test_removes_oldest_items_first(self): remove_count = QueryTaskTracker.prune(self.list, 50) self.assertEqual(remove_count, 50) self.assertEqual(redis_connection.zcard(self.list), 50) self.assertEqual(redis_connection.zscore(self.list, 'k:99'), 99.0) self.assertIsNone(redis_connection.zscore(self.list, 'k:1'))
def prune(cls, list_name, keep_count, max_keys=100): count = redis_connection.zcard(list_name) if count <= keep_count: return 0 remove_count = min(max_keys, count - keep_count) keys = redis_connection.zrange(list_name, 0, remove_count - 1) redis_connection.delete(*keys) redis_connection.zremrangebyrank(list_name, 0, remove_count - 1) return remove_count
def prune(cls, list_name, keep_count): count = redis_connection.zcard(list_name) if count <= keep_count: return 0 remove_count = count - keep_count keys = redis_connection.zrange(list_name, 0, remove_count - 1) redis_connection.delete(keys) redis_connection.zremrangebyrank(list_name, 0, remove_count - 1) return remove_count
def test_multiple_enqueue_of_same_query(self): query = self.factory.create_query() execute_query.apply_async = MagicMock(side_effect=gen_hash) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) self.assertEqual(1, execute_query.apply_async.call_count) self.assertEqual(1, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual( 0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def status_api(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [redis_connection.hgetall(w) for w in redis_connection.smembers('workers')] manager_status = redis_connection.hgetall('manager:status') status['manager'] = manager_status status['manager']['queue_size'] = redis_connection.zcard('jobs') return jsonify(status)
def status_api(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [ redis_connection.hgetall(w) for w in redis_connection.smembers('workers') ] manager_status = redis_connection.hgetall('manager:status') status['manager'] = manager_status status['manager']['queue_size'] = redis_connection.zcard('jobs') return jsonify(status)
def test_does_nothing_when_below_threshold(self): remove_count = QueryTaskTracker.prune(self.list, 100) self.assertEqual(remove_count, 0) self.assertEqual(redis_connection.zcard(self.list), 100)