def render_template(self, template): if template is None: return "" data = self.query_rel.latest_query_data.data host = base_url(self.query_rel.org) col_name = self.options["column"] if data["rows"] and col_name in data["rows"][0]: result_value = data["rows"][0][col_name] else: result_value = None context = { "ALERT_NAME": self.name, "ALERT_URL": "{host}/alerts/{alert_id}".format(host=host, alert_id=self.id), "ALERT_STATUS": self.state.upper(), "ALERT_CONDITION": self.options["op"], "ALERT_THRESHOLD": self.options["value"], "QUERY_NAME": self.query_rel.name, "QUERY_URL": "{host}/queries/{query_id}".format( host=host, query_id=self.query_rel.id ), "QUERY_RESULT_VALUE": result_value, "QUERY_RESULT_ROWS": data["rows"], "QUERY_RESULT_COLS": data["columns"], } return mustache_render(template, context)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format(", ".join(missing_params))) if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format(data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format(data_source.name) return error_response(message) if query_parameters: query_text = mustache_render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, current_user.id, metadata={"Username": current_user.email, "Query ID": query_id}) return {'job': job.to_dict()}
def post(self, data_source_id): params = request.get_json(force=True) parameter_values = collect_parameters_from_request(request.args) query = mustache_render(params['query'], parameter_values) data_source = get_object_or_404( models.DataSource.get_by_id_and_org, data_source_id, self.current_org, ) # get the validator method validator = self.get_validator(data_source.query_runner) if (validator is None or not has_access(data_source.groups, self.current_user, not_view_only)): return { 'valid': False, 'report': 'You do not have permission to run queries with this data source.' }, 403 try: valid, report = validator(data_source.query_runner, query) return {'valid': valid, 'report': report}, 200 except apiclient.errors.HttpError as e: if e.resp.status == 400: error = json_loads(e.content)['error']['message'] else: error = e.content return {'valid': False, 'report': error}, 200 except Exception as e: return {'valid': False, 'report': str(e)}, 500
def apply(self, parameters): invalid_parameter_names = [key for (key, value) in parameters.iteritems() if not self._valid(key, value)] if invalid_parameter_names: raise InvalidParameterError(invalid_parameter_names) else: self.parameters.update(parameters) self.query = mustache_render(self.template, self.parameters) return self
def render_template(self): if not self.template: return '' data = json_loads(self.query_rel.latest_query_data.data) context = { 'rows': data['rows'], 'cols': data['columns'], 'state': self.state } return mustache_render(self.template, context)
def run_query_sync(data_source, parameter_values, query_text, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: raise Exception('Missing parameter value for: {}'.format( ", ".join(missing_params))) if query_parameters: query_text = mustache_render(query_text, parameter_values) if max_age <= 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) query_hash = gen_query_hash(query_text) if query_result: logging.info("Returning cached result for query %s" % query_hash) return query_result try: started_at = time.time() data, error = data_source.query_runner.run_query( query_text, current_user) if error: logging.info('got bak error') logging.info(error) return None run_time = time.time() - started_at query_result, updated_query_ids = models.QueryResult.store_result( data_source.org_id, data_source, query_hash, query_text, data, run_time, utcnow()) models.db.session.commit() return query_result except Exception as e: if max_age > 0: abort( 404, message= "Unable to get result from the database, and no cached query result found." ) else: abort(503, message="Unable to get result from the database.") return None
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.info("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: if query.options and len(query.options.get('parameters', [])) > 0: query_params = {p['name']: p.get('value') for p in query.options['parameters']} query_text = mustache_render(query.query_text, query_params) else: query_text = query.query_text if is_enqueued(query_text, query.data_source.id): logging.info("Skipping refresh of %s because query is already queued up.", query.id) else: enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def execute_by_query_id(query_id, params=None): """Run query from specific query_id. Parameters: :query_id int: Query id to run :params dict: Params for bind to query """ query = get_query(query_id) query_text = query.query_text if params is None: query_text = query.query_text else: query_text = mustache_render(query.query_text, params) data, error = query.data_source.query_runner.run_query( query_text, None) if error is not None: raise Exception(error) return json_loads(data)
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.info("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: if query.options and len(query.options.get('parameters', [])) > 0: query_params = {p['name']: p.get('value') for p in query.options['parameters']} query_text = mustache_render(query.query_text, query_params) else: query_text = query.query_text enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def run_query_sync(data_source, parameter_values, query_text, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: raise Exception('Missing parameter value for: {}'.format(", ".join(missing_params))) if query_parameters: query_text = mustache_render(query_text, parameter_values) if max_age <= 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) query_hash = gen_query_hash(query_text) if query_result: logging.info("Returning cached result for query %s" % query_hash) return query_result try: started_at = time.time() data, error = data_source.query_runner.run_query(query_text, current_user) if error: logging.info('got bak error') logging.info(error) return None run_time = time.time() - started_at query_result, updated_query_ids = models.QueryResult.store_result(data_source.org_id, data_source, query_hash, query_text, data, run_time, utcnow()) models.db.session.commit() return query_result except Exception as e: if max_age > 0: abort(404, message="Unable to get result from the database, and no cached query result found.") else: abort(503, message="Unable to get result from the database.") return None
def render_template(self, template): if template is None: return '' data = self.query_rel.latest_query_data.data host = base_url(self.query_rel.org) col_name = self.options['column'] if data['rows'] and col_name in data['rows'][0]: result_value = data['rows'][0][col_name] else: result_value = None context = { 'ALERT_NAME': self.name, 'ALERT_URL': '{host}/alerts/{alert_id}'.format(host=host, alert_id=self.id), 'ALERT_STATUS': self.state.upper(), 'ALERT_CONDITION': self.options['op'], 'ALERT_THRESHOLD': self.options['value'], 'QUERY_NAME': self.query_rel.name, 'QUERY_URL': '{host}/queries/{query_id}'.format(host=host, query_id=self.query_rel.id), 'QUERY_RESULT_VALUE': result_value, 'QUERY_RESULT_ROWS': data['rows'], 'QUERY_RESULT_COLS': data['columns'], } return mustache_render(template, context)
def run_query(data_source, parameter_values, query_text, query_id, max_age=0): query_parameters = set(collect_query_parameters(query_text)) missing_params = set(query_parameters) - set(parameter_values.keys()) if missing_params: return error_response('Missing parameter value for: {}'.format( ", ".join(missing_params))) if data_source.paused: if data_source.pause_reason: message = '{} is paused ({}). Please try later.'.format( data_source.name, data_source.pause_reason) else: message = '{} is paused. Please try later.'.format( data_source.name) return error_response(message) if query_parameters: query_text = mustache_render(query_text, parameter_values) if max_age == 0: query_result = None else: query_result = models.QueryResult.get_latest(data_source, query_text, max_age) if query_result: return {'query_result': query_result.to_dict()} else: job = enqueue_query(query_text, data_source, current_user.id, metadata={ "Username": current_user.email, "Query ID": query_id }) return {'job': job.to_dict()}
def apply(self, parameters): self.query = mustache_render(self.template, parameters) return self
def get_saml_client(org): """ Return SAML configuration. The configuration is a hash for use by saml2.config.Config """ saml_type = org.get_setting("auth_saml_type") entity_id = org.get_setting("auth_saml_entity_id") sso_url = org.get_setting("auth_saml_sso_url") x509_cert = org.get_setting("auth_saml_x509_cert") metadata_url = org.get_setting("auth_saml_metadata_url") sp_settings = org.get_setting("auth_saml_sp_settings") if settings.SAML_SCHEME_OVERRIDE: acs_url = url_for( "saml_auth.idp_initiated", org_slug=org.slug, _external=True, _scheme=settings.SAML_SCHEME_OVERRIDE, ) else: acs_url = url_for("saml_auth.idp_initiated", org_slug=org.slug, _external=True) saml_settings = { "metadata": {"remote": [{"url": metadata_url}]}, "service": { "sp": { "endpoints": { "assertion_consumer_service": [ (acs_url, BINDING_HTTP_REDIRECT), (acs_url, BINDING_HTTP_POST), ] }, # Don't verify that the incoming requests originate from us via # the built-in cache for authn request ids in pysaml2 "allow_unsolicited": True, # Don't sign authn requests, since signed requests only make # sense in a situation where you control both the SP and IdP "authn_requests_signed": False, "logout_requests_signed": True, "want_assertions_signed": True, "want_response_signed": False, } }, } if settings.SAML_ENCRYPTION_ENABLED: encryption_dict = { "xmlsec_binary": get_xmlsec_binary(), "encryption_keypairs": [ { "key_file": settings.SAML_ENCRYPTION_PEM_PATH, "cert_file": settings.SAML_ENCRYPTION_CERT_PATH, } ], } saml_settings.update(encryption_dict) if saml_type is not None and saml_type == "static": metadata_inline = mustache_render( inline_metadata_template, entity_id=entity_id, x509_cert=x509_cert, sso_url=sso_url, ) saml_settings["metadata"] = {"inline": [metadata_inline]} if acs_url is not None and acs_url != "": saml_settings["entityid"] = acs_url if sp_settings: import json saml_settings["service"]["sp"].update(json.loads(sp_settings)) sp_config = Saml2Config() sp_config.load(saml_settings) sp_config.allow_unknown_attributes = True saml_client = Saml2Client(config=sp_config) return saml_client