def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return {
'job': {
'status':
4,
'error':
'Missing parameter value for: {}'.format(
", ".join(missing_params))
}
}, 400
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = QueryTask.add_task(query_text,
data_source,
metadata={
"Username": current_user.name,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, metadata={"Username": current_user.name, "Query ID": query_id})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id})
return {'job': job.to_dict()}
def run_query_sync(data_source, parameter_values, query_text, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
raise Exception('Missing parameter value for: {}'.format(", ".join(missing_params)))
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age <= 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
query_hash = gen_query_hash(query_text)
if query_result:
logging.info("Returning cached result for query %s" % query_hash)
return query_result
try:
started_at = time.time()
data, error = data_source.query_runner.run_query(query_text, current_user)
if error:
logging.info('got bak error')
logging.info(error)
data_consumed_mb = 0.0
return None
else:
dict_data = json.loads(data)
data_consumed_mb = dict_data['data_consumed_mb']
run_time = time.time() - started_at
query_result, updated_query_ids = models.QueryResult.store_result(data_source.org, data_source,
query_hash, query_text, data,
run_time, utils.utcnow())
query_result_metadata = models.QueryResultMetaData.store_result_metadata(
updated_query_ids=updated_query_ids,
query_result_id=query_result.id,
data_consumed_mb=data_consumed_mb,
data_source_id=data_source.id,
query_hash=query_hash,
run_by_user_id=current_user.id,
run_at=utils.utcnow())
models.db.session.commit()
return query_result
except Exception as e:
if max_age > 0:
abort(404, message="Unable to get result from the database, and no cached query result found.")
else:
abort(503, message="Unable to get result from the database.")
return None
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
# adds a job if max_age=0 -> /job and /event calls
# how is status=3 prompting /query_results/2 call ?
# anyway, query_result/2 call - model.py - gets final result through db.sessions
# how is celery job connected to db.session object ??
#return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age))
original_text = query_text
query_text = translate(query_text)
if query_text == 'NA':
query_text = original_text
#return error_response('New Query text variable is <{}>, param values is <{}>, query id is <{}>, max age is <{}>'.format(query_text,parameter_values,query_id, max_age))
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query_sync(data_source, parameter_values, query_text, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
raise Exception('Missing parameter value for: {}'.format(
", ".join(missing_params)))
query_text = apply_parameters(query_text, parameter_values, data_source)
if max_age <= 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
query_hash = gen_query_hash(query_text)
if query_result:
logging.info("Returning cached result for query %s" % query_hash)
return query_result
try:
started_at = time.time()
data, error = data_source.query_runner.run_query(
query_text, current_user)
if error:
logging.info('got bak error')
logging.info(error)
return None
run_time = time.time() - started_at
query_result, updated_query_ids = models.QueryResult.store_result(
data_source.org_id, data_source, query_hash, query_text, data,
run_time, utcnow())
models.db.session.commit()
return query_result
except Exception as e:
if max_age > 0:
abort(
404,
message=
"Unable to get result from the database, and no cached query result found."
)
else:
abort(503, message="Unable to get result from the database.")
return None
def run_query_sync(data_source, parameter_values, query_text, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
raise Exception('Missing parameter value for: {}'.format(", ".join(missing_params)))
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age <= 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
query_hash = gen_query_hash(query_text)
if query_result:
logging.info("Returning cached result for query %s" % query_hash)
return query_result
try:
started_at = time.time()
data, error = data_source.query_runner.run_query(query_text, current_user)
if error:
logging.info('got bak error')
logging.info(error)
return None
run_time = time.time() - started_at
query_result, updated_query_ids = models.QueryResult.store_result(data_source.org_id, data_source,
query_hash, query_text, data,
run_time, utcnow())
models.db.session.commit()
return query_result
except Exception as e:
if max_age > 0:
abort(404, message="Unable to get result from the database, and no cached query result found.")
else:
abort(503, message="Unable to get result from the database.")
return None
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
# 运行一个query
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if data_source.paused: # 数据源被停用
if data_source.pause_reason: # 数据源被停用的原因
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
# query_result如果是None，就执行新的查询，否则可能使用已有的查询
if max_age == 0:
query_result = None
else:
# 查看是否有符合条件的查询结果
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
# 直接使用刚刚的查询结果，不再发生新的查询
return {'query_result': query_result.to_dict()}
else:
# 重新查询 //先创建一个job
job = enqueue_query(query_text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return {'job': {'status': 4,
'error': 'Missing parameter value for: {}'.format(", ".join(missing_params))}}, 400
if query_parameters:
query_text = pystache.render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, metadata={"Username": current_user.name, "Query ID": query_id})
return {'job': job.to_dict()}
def test_handles_nested_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}} -- {{#test}} {{nested_param}} {{/test}}"
params = ['param', 'table', 'test', 'nested_param']
self.assertEqual(params, collect_query_parameters(query))
def test_deduplicates_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_finds_all_params(self):
query = u"SELECT {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_returns_empty_list_for_regular_query(self):
query = u"SELECT 1"
self.assertEqual([], collect_query_parameters(query))
def test_returns_empty_list_for_regular_query(self):
query = u"SELECT 1"
self.assertEqual([], collect_query_parameters(query))
def test_finds_all_params(self):
query = u"SELECT {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_deduplicates_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}}"
params = ['param', 'table']
self.assertEqual(params, collect_query_parameters(query))
def test_handles_nested_params(self):
query = u"SELECT {{param}}, {{param}} FROM {{table}} -- {{#test}} {{nested_param}} {{/test}}"
params = ['param', 'table', 'test', 'nested_param']
self.assertEqual(params, collect_query_parameters(query))