def render_template(self, template):
if template is None:
return ""
data = self.query_rel.latest_query_data.data
host = base_url(self.query_rel.org)
col_name = self.options["column"]
if data["rows"] and col_name in data["rows"][0]:
result_value = data["rows"][0][col_name]
else:
result_value = None
context = {
"ALERT_NAME": self.name,
"ALERT_URL": "{host}/alerts/{alert_id}".format(host=host, alert_id=self.id),
"ALERT_STATUS": self.state.upper(),
"ALERT_CONDITION": self.options["op"],
"ALERT_THRESHOLD": self.options["value"],
"QUERY_NAME": self.query_rel.name,
"QUERY_URL": "{host}/queries/{query_id}".format(
host=host, query_id=self.query_rel.id
),
"QUERY_RESULT_VALUE": result_value,
"QUERY_RESULT_ROWS": data["rows"],
"QUERY_RESULT_COLS": data["columns"],
}
return mustache_render(template, context)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(data_source.name)
return error_response(message)
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id})
return {'job': job.to_dict()}
def post(self, data_source_id):
params = request.get_json(force=True)
parameter_values = collect_parameters_from_request(request.args)
query = mustache_render(params['query'], parameter_values)
data_source = get_object_or_404(
models.DataSource.get_by_id_and_org,
data_source_id,
self.current_org,
)
# get the validator method
validator = self.get_validator(data_source.query_runner)
if (validator is None
or not has_access(data_source.groups, self.current_user, not_view_only)):
return {
'valid': False,
'report': 'You do not have permission to run queries with this data source.'
}, 403
try:
valid, report = validator(data_source.query_runner, query)
return {'valid': valid, 'report': report}, 200
except apiclient.errors.HttpError as e:
if e.resp.status == 400:
error = json_loads(e.content)['error']['message']
else:
error = e.content
return {'valid': False, 'report': error}, 200
except Exception as e:
return {'valid': False, 'report': str(e)}, 500
def apply(self, parameters):
invalid_parameter_names = [key for (key, value) in parameters.iteritems() if not self._valid(key, value)]
if invalid_parameter_names:
raise InvalidParameterError(invalid_parameter_names)
else:
self.parameters.update(parameters)
self.query = mustache_render(self.template, self.parameters)
return self
def apply(self, parameters):
invalid_parameter_names = [key for (key, value) in parameters.iteritems() if not self._valid(key, value)]
if invalid_parameter_names:
raise InvalidParameterError(invalid_parameter_names)
else:
self.parameters.update(parameters)
self.query = mustache_render(self.template, self.parameters)
return self
def render_template(self):
if not self.template:
return ''
data = json_loads(self.query_rel.latest_query_data.data)
context = {
'rows': data['rows'],
'cols': data['columns'],
'state': self.state
}
return mustache_render(self.template, context)
def run_query_sync(data_source, parameter_values, query_text, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
raise Exception('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age <= 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
query_hash = gen_query_hash(query_text)
if query_result:
logging.info("Returning cached result for query %s" % query_hash)
return query_result
try:
started_at = time.time()
data, error = data_source.query_runner.run_query(
query_text, current_user)
if error:
logging.info('got bak error')
logging.info(error)
return None
run_time = time.time() - started_at
query_result, updated_query_ids = models.QueryResult.store_result(
data_source.org_id, data_source, query_hash, query_text, data,
run_time, utcnow())
models.db.session.commit()
return query_result
except Exception as e:
if max_age > 0:
abort(
404,
message=
"Unable to get result from the database, and no cached query result found."
)
else:
abort(503, message="Unable to get result from the database.")
return None
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if settings.FEATURE_DISABLE_REFRESH_QUERIES:
logging.info("Disabled refresh queries.")
elif query.org.is_disabled:
logging.debug("Skipping refresh of %s because org is disabled.", query.id)
elif query.data_source is None:
logging.info("Skipping refresh of %s because the datasource is none.", query.id)
elif query.data_source.paused:
logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason)
else:
if query.options and len(query.options.get('parameters', [])) > 0:
query_params = {p['name']: p.get('value')
for p in query.options['parameters']}
query_text = mustache_render(query.query_text, query_params)
else:
query_text = query.query_text
if is_enqueued(query_text, query.data_source.id):
logging.info("Skipping refresh of %s because query is already queued up.", query.id)
else:
enqueue_query(query_text, query.data_source, query.user_id,
scheduled_query=query,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json_dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def execute_by_query_id(query_id, params=None):
"""Run query from specific query_id.
Parameters:
:query_id int: Query id to run
:params dict: Params for bind to query
"""
query = get_query(query_id)
query_text = query.query_text
if params is None:
query_text = query.query_text
else:
query_text = mustache_render(query.query_text, params)
data, error = query.data_source.query_runner.run_query(
query_text, None)
if error is not None:
raise Exception(error)
return json_loads(data)
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if settings.FEATURE_DISABLE_REFRESH_QUERIES:
logging.info("Disabled refresh queries.")
elif query.org.is_disabled:
logging.debug("Skipping refresh of %s because org is disabled.", query.id)
elif query.data_source is None:
logging.info("Skipping refresh of %s because the datasource is none.", query.id)
elif query.data_source.paused:
logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason)
else:
if query.options and len(query.options.get('parameters', [])) > 0:
query_params = {p['name']: p.get('value')
for p in query.options['parameters']}
query_text = mustache_render(query.query_text, query_params)
else:
query_text = query.query_text
enqueue_query(query_text, query.data_source, query.user_id,
scheduled_query=query,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json_dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def run_query_sync(data_source, parameter_values, query_text, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
raise Exception('Missing parameter value for: {}'.format(", ".join(missing_params)))
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age <= 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text, max_age)
query_hash = gen_query_hash(query_text)
if query_result:
logging.info("Returning cached result for query %s" % query_hash)
return query_result
try:
started_at = time.time()
data, error = data_source.query_runner.run_query(query_text, current_user)
if error:
logging.info('got bak error')
logging.info(error)
return None
run_time = time.time() - started_at
query_result, updated_query_ids = models.QueryResult.store_result(data_source.org_id, data_source,
query_hash, query_text, data,
run_time, utcnow())
models.db.session.commit()
return query_result
except Exception as e:
if max_age > 0:
abort(404, message="Unable to get result from the database, and no cached query result found.")
else:
abort(503, message="Unable to get result from the database.")
return None
def render_template(self, template):
if template is None:
return ''
data = self.query_rel.latest_query_data.data
host = base_url(self.query_rel.org)
col_name = self.options['column']
if data['rows'] and col_name in data['rows'][0]:
result_value = data['rows'][0][col_name]
else:
result_value = None
context = {
'ALERT_NAME':
self.name,
'ALERT_URL':
'{host}/alerts/{alert_id}'.format(host=host, alert_id=self.id),
'ALERT_STATUS':
self.state.upper(),
'ALERT_CONDITION':
self.options['op'],
'ALERT_THRESHOLD':
self.options['value'],
'QUERY_NAME':
self.query_rel.name,
'QUERY_URL':
'{host}/queries/{query_id}'.format(host=host,
query_id=self.query_rel.id),
'QUERY_RESULT_VALUE':
result_value,
'QUERY_RESULT_ROWS':
data['rows'],
'QUERY_RESULT_COLS':
data['columns'],
}
return mustache_render(template, context)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0):
query_parameters = set(collect_query_parameters(query_text))
missing_params = set(query_parameters) - set(parameter_values.keys())
if missing_params:
return error_response('Missing parameter value for: {}'.format(
", ".join(missing_params)))
if data_source.paused:
if data_source.pause_reason:
message = '{} is paused ({}). Please try later.'.format(
data_source.name, data_source.pause_reason)
else:
message = '{} is paused. Please try later.'.format(
data_source.name)
return error_response(message)
if query_parameters:
query_text = mustache_render(query_text, parameter_values)
if max_age == 0:
query_result = None
else:
query_result = models.QueryResult.get_latest(data_source, query_text,
max_age)
if query_result:
return {'query_result': query_result.to_dict()}
else:
job = enqueue_query(query_text,
data_source,
current_user.id,
metadata={
"Username": current_user.email,
"Query ID": query_id
})
return {'job': job.to_dict()}
def apply(self, parameters):
self.query = mustache_render(self.template, parameters)
return self
def get_saml_client(org):
"""
Return SAML configuration.
The configuration is a hash for use by saml2.config.Config
"""
saml_type = org.get_setting("auth_saml_type")
entity_id = org.get_setting("auth_saml_entity_id")
sso_url = org.get_setting("auth_saml_sso_url")
x509_cert = org.get_setting("auth_saml_x509_cert")
metadata_url = org.get_setting("auth_saml_metadata_url")
sp_settings = org.get_setting("auth_saml_sp_settings")
if settings.SAML_SCHEME_OVERRIDE:
acs_url = url_for(
"saml_auth.idp_initiated",
org_slug=org.slug,
_external=True,
_scheme=settings.SAML_SCHEME_OVERRIDE,
)
else:
acs_url = url_for("saml_auth.idp_initiated", org_slug=org.slug, _external=True)
saml_settings = {
"metadata": {"remote": [{"url": metadata_url}]},
"service": {
"sp": {
"endpoints": {
"assertion_consumer_service": [
(acs_url, BINDING_HTTP_REDIRECT),
(acs_url, BINDING_HTTP_POST),
]
},
# Don't verify that the incoming requests originate from us via
# the built-in cache for authn request ids in pysaml2
"allow_unsolicited": True,
# Don't sign authn requests, since signed requests only make
# sense in a situation where you control both the SP and IdP
"authn_requests_signed": False,
"logout_requests_signed": True,
"want_assertions_signed": True,
"want_response_signed": False,
}
},
}
if settings.SAML_ENCRYPTION_ENABLED:
encryption_dict = {
"xmlsec_binary": get_xmlsec_binary(),
"encryption_keypairs": [
{
"key_file": settings.SAML_ENCRYPTION_PEM_PATH,
"cert_file": settings.SAML_ENCRYPTION_CERT_PATH,
}
],
}
saml_settings.update(encryption_dict)
if saml_type is not None and saml_type == "static":
metadata_inline = mustache_render(
inline_metadata_template,
entity_id=entity_id,
x509_cert=x509_cert,
sso_url=sso_url,
)
saml_settings["metadata"] = {"inline": [metadata_inline]}
if acs_url is not None and acs_url != "":
saml_settings["entityid"] = acs_url
if sp_settings:
import json
saml_settings["service"]["sp"].update(json.loads(sp_settings))
sp_config = Saml2Config()
sp_config.load(saml_settings)
sp_config.allow_unknown_attributes = True
saml_client = Saml2Client(config=sp_config)
return saml_client
