def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
if current_user.is_api_user():
job = enqueue_query(query_text, data_source, None, metadata={"Query ID": query_id})
else:
job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id})
return {'job': job.to_dict()}
def test_limits_query_time(self, _):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, False, query, {'Username': '******', 'Query ID': query.id})
_, kwargs = execute_query.apply_async.call_args
self.assertEqual(60, kwargs.get('soft_time_limit'))
def test_limits_query_time(self, _):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, False, query, {'Username': '******', 'Query ID': query.id})
_, kwargs = execute_query.apply_async.call_args
self.assertEqual(60, kwargs.get('time_limit'))
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return {
'job': {
'status':
4,
'error':
'Missing parameter value for: {}'.format(
", ".join(missing_params))
}
}, 400
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text,
data_source,
metadata={
"Username": current_user.name,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
query = ParameterizedQuery(query_text).apply(parameter_values)
if query.missing_params:
return error_response(u'Missing parameter value for: {}'.format(
u", ".join(query.missing_params)))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query.text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query.text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0):
if data_source.paused:
if data_source.pause_reason:
message = "{} is paused ({}). Please try later.".format(
data_source.name, data_source.pause_reason)
else:
message = "{} is paused. Please try later.".format(
data_source.name)
return error_response(message)
try:
query.apply(parameters)
except (InvalidParameterError, QueryDetachedFromDataSourceError) as e:
abort(400, message=str(e))
if query.missing_params:
return error_response("Missing parameter value for: {}".format(
", ".join(query.missing_params)))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query.text,
max_age)
record_event(
current_user.org,
current_user,
{
"action": "execute_query",
"cache": "hit" if query_result else "miss",
"object_id": data_source.id,
"object_type": "data_source",
"query": query.text,
"query_id": query_id,
"parameters": parameters,
},
)
if query_result:
return {
"query_result":
serialize_query_result(query_result, current_user.is_api_user())
}
else:
job = enqueue_query(
query.text,
data_source,
current_user.id,
current_user.is_api_user(),
metadata={
"Username":
repr(current_user)
if current_user.is_api_user() else current_user.email,
"Query ID":
query_id,
},
)
return serialize_job(job)
def run_query(query, parameters, data_source, query_id, max_age=0):
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
try:
query.apply(parameters)
except InvalidParameterError as e:
abort(400, message=e.message)
if query.missing_params:
return error_response(u'Missing parameter value for: {}'.format(u", ".join(query.missing_params)))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query.text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query.text, data_source, current_user.id, current_user.is_api_user(), metadata={
"Username": repr(current_user) if current_user.is_api_user() else current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0):
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
try:
query.apply(parameters)
except InvalidParameterError as e:
abort(400, message=e.message)
if query.missing_params:
return error_response(u'Missing parameter value for: {}'.format(u", ".join(query.missing_params)))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query.text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query.text, data_source, current_user.id, current_user.is_api_user(), metadata={
"Username": repr(current_user) if current_user.is_api_user() else current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0):
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
try:
query.apply(parameters)
except (InvalidParameterError, QueryDetachedFromDataSourceError) as e:
abort(400, message=e.message)
if query.missing_params:
return error_response(u'Missing parameter value for: {}'.format(
u", ".join(query.missing_params)))
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query.text,
max_age)
record_event(
current_user.org, current_user, {
'action': 'execute_query',
'cache': 'hit' if query_result else 'miss',
'object_id': data_source.id,
'object_type': 'data_source',
'query': query.text,
'query_id': query_id,
'parameters': parameters
})
if query_result:
return {
'query_result':
serialize_query_result(query_result, current_user.is_api_user())
}
else:
job = enqueue_query(
query.text,
data_source,
current_user.id,
current_user.is_api_user(),
metadata={
"Username":
repr(current_user)
if current_user.is_api_user() else current_user.email,
"Query ID":
query_id
})
return {'job': job.to_dict()}
def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, None,
{
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query_text + '2', query.data_source, query.user_id,
None, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query_text + '3', query.data_source, query.user_id,
None, {
'Username': '******',
'Query ID': query.id
})
self.assertEqual(3, execute_query.apply_async.call_count)
self.assertEqual(3,
redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(
0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def enqueue_query(query, *a, **kw):
from redash.tasks.queries import enqueue_query
job = enqueue_query(query, *a, **kw)
tasks.append(
dict(
state='waiting_in_queue',
task_name='test task',
worker=None,
worker_pid=None,
start_time=None,
task_id=job.id,
queue='queries',
))
return job
def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query_text + '2', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query_text + '3', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
self.assertEqual(3, execute_query.apply_async.call_count)
def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = mock.MagicMock(side_effect=gen_hash)
enqueue_query(query.query_text, query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query_text + '2', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query_text + '3', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id})
self.assertEqual(3, execute_query.apply_async.call_count)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
# adds a job if max_age=0 -> /job and /event calls
# how is status=3 prompting /query_results/2 call ?
# anyway, query_result/2 call - model.py - gets final result through db.sessions
# how is celery job connected to db.session object ??
#return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age))
original_text = query_text
query_text = translate(query_text)
if query_text == 'NA':
query_text = original_text
#return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age))
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def test_multiple_enqueue_of_different_query(self):
query = self.factory.create_query()
execute_query.apply_async = MagicMock(side_effect=gen_hash)
enqueue_query(query.query, query.data_source, True, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query + '2', query.data_source, True, {'Username': '******', 'Query ID': query.id})
enqueue_query(query.query + '3', query.data_source, True, {'Username': '******', 'Query ID': query.id})
self.assertEqual(3, execute_query.apply_async.call_count)
self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
# 运行一个query
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if data_source.paused: # 数据源被停用
if data_source.pause_reason: # 数据源被停用的原因
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
# query_result如果是None，就执行新的查询，否则可能使用已有的查询
if max_age == 0:
query_result = None
else:
# 查看是否有符合条件的查询结果
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
# 直接使用刚刚的查询结果，不再发生新的查询
return {'query_result': query_result.to_dict()}
else:
# 重新查询 //先创建一个job
job = enqueue_query(query_text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return {'job': {'status': 4,
'error': 'Missing parameter value for: {}'.format(", ".join(missing_params))}}, 400
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, metadata={"Username": current_user.name, "Query ID": query_id})
return {'job': job.to_dict()}
def test_multiple_enqueue_of_same_query(self):
query = self.factory.create_query()
execute_query.apply_async = MagicMock(side_effect=gen_hash)
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
enqueue_query(query.query, query.data_source, True, {
'Username': '******',
'Query ID': query.id
})
self.assertEqual(1, execute_query.apply_async.call_count)
self.assertEqual(1,
redis_connection.zcard(QueryTaskTracker.WAITING_LIST))
self.assertEqual(
0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST))
self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
