def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format(", ".join(missing_params))) if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format(data_source.name) return error_response(message) if query_parameters: query_text = pystache.render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: if current_user.is_api_user(): job = enqueue_query(query_text, data_source, None, metadata={"Query ID": query_id}) else: job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id}) return {'job': job.to_dict()}
def test_limits_query_time(self, _): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, False, query, {'Username': '******', 'Query ID': query.id}) _, kwargs = execute_query.apply_async.call_args self.assertEqual(60, kwargs.get('soft_time_limit'))
def test_limits_query_time(self, _): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, False, query, {'Username': '******', 'Query ID': query.id}) _, kwargs = execute_query.apply_async.call_args self.assertEqual(60, kwargs.get('time_limit'))
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return { 'job': { 'status': 4, 'error': 'Missing parameter value for: {}'.format( ", ".join(missing_params)) } }, 400 if query_parameters: query_text = pystache.render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, metadata={ "Username": current_user.name, "Query ID": query_id }) return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format(", ".join(missing_params))) if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format(data_source.name) return error_response(message) if query_parameters: query_text = mustache_render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id}) return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format( data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format( data_source.name) return error_response(message) query = ParameterizedQuery(query_text).apply(parameter_values) if query.missing_params: return error_response(u'Missing parameter value for: {}'.format( u", ".join(query.missing_params))) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query.text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query.text, data_source, current_user.id, metadata={ "Username": current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0): if data_source.paused: if data_source.pause_reason: message = "{} is paused ({}). Please try later.".format( data_source.name, data_source.pause_reason) else: message = "{} is paused. Please try later.".format( data_source.name) return error_response(message) try: query.apply(parameters) except (InvalidParameterError, QueryDetachedFromDataSourceError) as e: abort(400, message=str(e)) if query.missing_params: return error_response("Missing parameter value for: {}".format( ", ".join(query.missing_params))) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query.text, max_age) record_event( current_user.org, current_user, { "action": "execute_query", "cache": "hit" if query_result else "miss", "object_id": data_source.id, "object_type": "data_source", "query": query.text, "query_id": query_id, "parameters": parameters, }, ) if query_result: return { "query_result": serialize_query_result(query_result, current_user.is_api_user()) } else: job = enqueue_query( query.text, data_source, current_user.id, current_user.is_api_user(), metadata={ "Username": repr(current_user) if current_user.is_api_user() else current_user.email, "Query ID": query_id, }, ) return serialize_job(job)
def run_query(query, parameters, data_source, query_id, max_age=0): if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format(data_source.name) return error_response(message) try: query.apply(parameters) except InvalidParameterError as e: abort(400, message=e.message) if query.missing_params: return error_response(u'Missing parameter value for: {}'.format(u", ".join(query.missing_params))) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query.text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query.text, data_source, current_user.id, current_user.is_api_user(), metadata={ "Username": repr(current_user) if current_user.is_api_user() else current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0): if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format(data_source.name) return error_response(message) try: query.apply(parameters) except InvalidParameterError as e: abort(400, message=e.message) if query.missing_params: return error_response(u'Missing parameter value for: {}'.format(u", ".join(query.missing_params))) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query.text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query.text, data_source, current_user.id, current_user.is_api_user(), metadata={ "Username": repr(current_user) if current_user.is_api_user() else current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def run_query(query, parameters, data_source, query_id, max_age=0): if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format( data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format( data_source.name) return error_response(message) try: query.apply(parameters) except (InvalidParameterError, QueryDetachedFromDataSourceError) as e: abort(400, message=e.message) if query.missing_params: return error_response(u'Missing parameter value for: {}'.format( u", ".join(query.missing_params))) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query.text, max_age) record_event( current_user.org, current_user, { 'action': 'execute_query', 'cache': 'hit' if query_result else 'miss', 'object_id': data_source.id, 'object_type': 'data_source', 'query': query.text, 'query_id': query_id, 'parameters': parameters }) if query_result: return { 'query_result': serialize_query_result(query_result, current_user.is_api_user()) } else: job = enqueue_query( query.text, data_source, current_user.id, current_user.is_api_user(), metadata={ "Username": repr(current_user) if current_user.is_api_user() else current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query_text + '2', query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query_text + '3', query.data_source, query.user_id, None, { 'Username': '******', 'Query ID': query.id }) self.assertEqual(3, execute_query.apply_async.call_count) self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual( 0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def enqueue_query(query, *a, **kw): from redash.tasks.queries import enqueue_query job = enqueue_query(query, *a, **kw) tasks.append( dict( state='waiting_in_queue', task_name='test task', worker=None, worker_pid=None, start_time=None, task_id=job.id, queue='queries', )) return job
def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query_text + '2', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query_text + '3', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) self.assertEqual(3, execute_query.apply_async.call_count)
def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = mock.MagicMock(side_effect=gen_hash) enqueue_query(query.query_text, query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query_text + '2', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query_text + '3', query.data_source, query.user_id, False, None, {'Username': '******', 'Query ID': query.id}) self.assertEqual(3, execute_query.apply_async.call_count)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): # adds a job if max_age=0 -> /job and /event calls # how is status=3 prompting /query_results/2 call ? # anyway, query_result/2 call - model.py - gets final result through db.sessions # how is celery job connected to db.session object ?? #return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age)) original_text = query_text query_text = translate(query_text) if query_text == 'NA': query_text = original_text #return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age)) query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format( ", ".join(missing_params))) if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format( data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format( data_source.name) return error_response(message) if query_parameters: query_text = pystache.render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, current_user.id, metadata={ "Username": current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def test_multiple_enqueue_of_different_query(self): query = self.factory.create_query() execute_query.apply_async = MagicMock(side_effect=gen_hash) enqueue_query(query.query, query.data_source, True, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query + '2', query.data_source, True, {'Username': '******', 'Query ID': query.id}) enqueue_query(query.query + '3', query.data_source, True, {'Username': '******', 'Query ID': query.id}) self.assertEqual(3, execute_query.apply_async.call_count) self.assertEqual(3, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): # 运行一个query query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format( ", ".join(missing_params))) if data_source.paused: # 数据源被停用 if data_source.pause_reason: # 数据源被停用的原因 message = '{} is paused ({}). Please try later.'.format( data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format( data_source.name) return error_response(message) if query_parameters: query_text = pystache.render(query_text, parameter_values) # query_result如果是None,就执行新的查询,否则可能使用已有的查询 if max_age == 0: query_result = None else: # 查看是否有符合条件的查询结果 query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: # 直接使用刚刚的查询结果,不再发生新的查询 return {'query_result': query_result.to_dict()} else: # 重新查询 //先创建一个job job = enqueue_query(query_text, data_source, current_user.id, metadata={ "Username": current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return {'job': {'status': 4, 'error': 'Missing parameter value for: {}'.format(", ".join(missing_params))}}, 400 if query_parameters: query_text = pystache.render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, metadata={"Username": current_user.name, "Query ID": query_id}) return {'job': job.to_dict()}
def test_multiple_enqueue_of_same_query(self): query = self.factory.create_query() execute_query.apply_async = MagicMock(side_effect=gen_hash) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) enqueue_query(query.query, query.data_source, True, { 'Username': '******', 'Query ID': query.id }) self.assertEqual(1, execute_query.apply_async.call_count) self.assertEqual(1, redis_connection.zcard(QueryTaskTracker.WAITING_LIST)) self.assertEqual( 0, redis_connection.zcard(QueryTaskTracker.IN_PROGRESS_LIST)) self.assertEqual(0, redis_connection.zcard(QueryTaskTracker.DONE_LIST))