def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): # TODO: this should go into lower priority QueryTask.add_task(query.query, query.data_source, scheduled=True) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) # TODO: decide if we still need this # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs')) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def get_status(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['version'] = __version__ status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['unused_query_results_count'] = models.QueryResult.unused().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [] manager_status = redis_connection.hgetall('redash:status') status['manager'] = manager_status status['manager']['outdated_queries_count'] = len( models.Query.outdated_queries()) queues = {} for ds in models.DataSource.select(): for queue in (ds.queue_name, ds.scheduled_queue_name): queues.setdefault(queue, set()) queues[queue].add(ds.name) status['manager']['queues'] = {} for queue, sources in queues.iteritems(): status['manager']['queues'][queue] = { 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } return status
def get_status(): status = {} info = redis_connection.info() status['Redis使用内存'] = info['used_memory_human'] status['版本号'] = __version__ status['查询数'] = models.db.session.query(models.Query).count() if settings.FEATURE_SHOW_QUERY_RESULTS_COUNT: status['查询结果数'] = models.db.session.query(models.QueryResult).count() status['未使用查询结果数'] = models.QueryResult.unused().count() status['仪表盘数'] = models.Dashboard.query.count() status['控件数'] = models.Widget.query.count() status['workers'] = [] manager_status = redis_connection.hgetall('redash:status') status['manager'] = manager_status status['manager']['outdated_queries_count'] = len( models.Query.outdated_queries()) queues = {} for ds in models.DataSource.query: for queue in (ds.queue_name, ds.scheduled_queue_name): queues.setdefault(queue, set()) queues[queue].add(ds.name) status['manager']['queues'] = {} for queue, sources in queues.iteritems(): status['manager']['queues'][queue] = { 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } return status
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json_loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = ( models.Query.query.outerjoin(models.QueryResult) .filter(models.Query.id.in_(query_ids)) .order_by(models.Query.created_at.desc()) ) else: outdated_queries = [] record_event(current_org, current_user._get_current_object(), { 'action': 'list', 'object_type': 'outdated_queries', }) updated_at = None if manager_status and manager_status['last_refresh_at']: updated_at = manager_status['last_refresh_at'] response = { 'queries': QuerySerializer(outdated_queries, with_stats=True, with_last_modified_by=False).serialize(), 'updated_at': updated_at, } return json_response(response)
def outdated_queries(): manager_status = redis_connection.hgetall("redash:status") query_ids = json_loads(manager_status.get("query_ids", "[]")) if query_ids: outdated_queries = (models.Query.query.outerjoin( models.QueryResult).filter( models.Query.id.in_(query_ids)).order_by( models.Query.created_at.desc())) else: outdated_queries = [] record_event( current_org, current_user._get_current_object(), { "action": "list", "object_type": "outdated_queries", }, ) response = { "queries": QuerySerializer(outdated_queries, with_stats=True, with_last_modified_by=False).serialize(), "updated_at": manager_status["last_refresh_at"], } return json_response(response)
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task(query.query, query.data_source, scheduled=True, metadata={ 'Query ID': query.id, 'Username': '******' }) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def get_status(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['version'] = __version__ status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['unused_query_results_count'] = models.QueryResult.unused().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [] manager_status = redis_connection.hgetall('redash:status') status['manager'] = manager_status status['manager']['outdated_queries_count'] = len(models.Query.outdated_queries()) queues = {} for ds in models.DataSource.select(): for queue in (ds.queue_name, ds.scheduled_queue_name): queues.setdefault(queue, set()) queues[queue].add(ds.name) status['manager']['queues'] = {} for queue, sources in queues.iteritems(): status['manager']['queues'][queue] = { 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } return status
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): enqueue_query(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def get_status(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['version'] = __version__ status['queries_count'] = models.db.session.query(models.Query).count() if settings.FEATURE_SHOW_QUERY_RESULTS_COUNT: status['query_results_count'] = models.db.session.query(models.QueryResult).count() status['unused_query_results_count'] = models.QueryResult.unused().count() status['dashboards_count'] = models.Dashboard.query.count() status['widgets_count'] = models.Widget.query.count() status['workers'] = [] status['manager'] = redis_connection.hgetall('redash:status') queues = {} for ds in models.DataSource.query: for queue in (ds.queue_name, ds.scheduled_queue_name): queues.setdefault(queue, set()) queues[queue].add(ds.name) status['manager']['queues'] = {} for queue, sources in queues.iteritems(): status['manager']['queues'][queue] = { 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } return status
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: enqueue_query(query.query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def outdated_queries(): ### redis-----hash get all manager_status = redis_connection.hgetall( 'redash:status' ) # "Return a Python dict of the hash's name/value pairs" query_ids = json.loads(manager_status.get( 'query_ids', '[]')) # redis Hash结构里 嵌套了一个list,被json序列化为了字符串 if query_ids: ###用的了 in_ 和 order_by 和 desc , outerJoin # https://www.xncoding.com/2016/03/07/python/sqlalchemy02.html # 定义好了外键,系统就知道如何去关联 outdated_queries = (models.db.session.query( models.Query).outerjoin(models.QueryResult).filter( models.Query.id.in_(query_ids)).order_by( models.Query.created_at.desc())) else: outdated_queries = [] return json_response( dict(queries=[ q.to_dict(with_stats=True, with_last_modified_by=False) for q in outdated_queries ], updated_at=manager_status['last_refresh_at']))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: enqueue_query(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json.loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = (models.db.session.query( models.Query).outerjoin(models.QueryResult).filter( models.Query.id.in_(query_ids)).order_by( models.Query.created_at.desc())) else: outdated_queries = [] record_event( current_org, current_user, { 'action': 'view', 'object_type': 'api_call', 'object_id': 'admin/outdated_queries', 'timestamp': int(time.time()), }) return json_response( dict(queries=[ q.to_dict(with_stats=True, with_last_modified_by=False) for q in outdated_queries ], updated_at=manager_status['last_refresh_at']))
def status_api(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [redis_connection.hgetall(w) for w in redis_connection.smembers('workers')] manager_status = redis_connection.hgetall('manager:status') status['manager'] = manager_status status['manager']['queue_size'] = redis_connection.zcard('jobs') return jsonify(status)
def get_status(): status = {"version": __version__, "workers": []} status.update(get_redis_status()) status.update(get_object_counts()) status["manager"] = redis_connection.hgetall("redash:status") status["manager"]["queues"] = get_queues_status() status["database_metrics"] = {} status["database_metrics"]["metrics"] = get_db_sizes() return status
def get_status(): status = {'version': __version__, 'workers': []} status.update(get_redis_status()) status.update(get_object_counts()) status['manager'] = redis_connection.hgetall('redash:status') status['manager']['queues'] = get_queues_status() status['database_metrics'] = {} status['database_metrics']['metrics'] = get_db_sizes() return status
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.debug("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.debug("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: query_text = query.query_text parameters = {p['name']: p.get('value') for p in query.parameters} if any(parameters): try: query_text = query.parameterized.apply(parameters).query except InvalidParameterError as e: error = u"Skipping refresh of {} because of invalid parameters: {}".format(query.id, e.message) track_failure(query, error) continue except QueryDetachedFromDataSourceError as e: error = ("Skipping refresh of {} because a related dropdown " "query ({}) is unattached to any datasource.").format(query.id, e.query_id) track_failure(query, error) continue enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def status_api(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory_human'] status['queries_count'] = models.Query.select().count() status['query_results_count'] = models.QueryResult.select().count() status['dashboards_count'] = models.Dashboard.select().count() status['widgets_count'] = models.Widget.select().count() status['workers'] = [ redis_connection.hgetall(w) for w in redis_connection.smembers('workers') ] manager_status = redis_connection.hgetall('manager:status') status['manager'] = manager_status status['manager']['queue_size'] = redis_connection.zcard('jobs') return jsonify(status)
def get_status(): status = {} info = redis_connection.info() status['redis_used_memory'] = info['used_memory'] status['redis_used_memory_human'] = info['used_memory_human'] status['version'] = __version__ status['queries_count'] = models.db.session.query(models.Query).count() if settings.FEATURE_SHOW_QUERY_RESULTS_COUNT: status['query_results_count'] = models.db.session.query( models.QueryResult).count() status['unused_query_results_count'] = models.QueryResult.unused( ).count() status['dashboards_count'] = models.Dashboard.query.count() status['widgets_count'] = models.Widget.query.count() status['workers'] = [] status['manager'] = redis_connection.hgetall('redash:status') queues = {} for ds in models.DataSource.query: for queue in (ds.queue_name, ds.scheduled_queue_name): queues.setdefault(queue, set()) queues[queue].add(ds.name) status['manager']['queues'] = {} for queue, sources in queues.iteritems(): status['manager']['queues'][queue] = { 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } status['manager']['queues']['celery'] = { 'size': redis_connection.llen('celery'), 'data_sources': '' } status['database_metrics'] = [] # have to include the fake FROM in the SQL to prevent an IndexError queries = [ [ 'Query Results Size', "pg_size_pretty(pg_total_relation_size('query_results')) as size from (select 1) as a" ], [ 'Redash DB Size', "pg_size_pretty(pg_database_size('postgres')) as size from (select 1) as a" ] ] for query_name, query in queries: result = models.db.session.query(query).first() status['database_metrics'].append([query_name, result[0]]) return status
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json.loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = models.Query.select(models.Query, models.QueryResult.retrieved_at, models.QueryResult.runtime) \ .join(models.QueryResult, join_type=models.peewee.JOIN_LEFT_OUTER) \ .where(models.Query.id << query_ids) \ .order_by(models.Query.created_at.desc()) else: outdated_queries = [] return json_response(dict(queries=[q.to_dict(with_stats=True, with_last_modified_by=False) for q in outdated_queries], updated_at=manager_status['last_refresh_at']))
def get_status(): status = { 'version': __version__, 'workers': [] } status.update(get_redis_status()) status.update(get_object_counts()) status['manager'] = redis_connection.hgetall('redash:status') status['manager']['queues'] = get_queues_status() status['database_metrics'] = {} status['database_metrics']['metrics'] = get_db_sizes() return status
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json.loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = (models.db.session.query(models.Query) .outerjoin(models.QueryResult) .filter(models.Query.id.in_(query_ids)) .order_by(models.Query.created_at.desc())) else: outdated_queries = [] return json_response( dict(queries=[q.to_dict(with_stats=True, with_last_modified_by=False) for q in outdated_queries], updated_at=manager_status['last_refresh_at']))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.info("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: if query.options and len(query.options.get('parameters', [])) > 0: query_params = {p['name']: p.get('value') for p in query.options['parameters']} query_text = mustache_render(query.query_text, query_params) else: query_text = query.query_text if is_enqueued(query_text, query.data_source.id): logging.info("Skipping refresh of %s because query is already queued up.", query.id) else: enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json.loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = ( models.Query.query.outerjoin(models.QueryResult) .filter(models.Query.id.in_(query_ids)) .order_by(models.Query.created_at.desc()) ) else: outdated_queries = [] response = { 'queries': QuerySerializer(outdated_queries, with_stats=True, with_last_modified_by=False).serialize(), 'updated_at': manager_status['last_refresh_at'], } return json_response(response)
def outdated_queries(): manager_status = redis_connection.hgetall('redash:status') query_ids = json.loads(manager_status.get('query_ids', '[]')) if query_ids: outdated_queries = (models.Query.query.outerjoin( models.QueryResult).filter( models.Query.id.in_(query_ids)).order_by( models.Query.created_at.desc())) else: outdated_queries = [] response = { 'queries': QuerySerializer(outdated_queries, with_stats=True, with_last_modified_by=False).serialize(), 'updated_at': manager_status['last_refresh_at'], } return json_response(response)
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task( query.query, query.data_source, scheduled=True, metadata={"Query ID": query.id, "Username": "******"} ) outdated_queries_count += 1 statsd_client.gauge("manager.outdated_queries", outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall("redash:status") now = time.time() redis_connection.hmset("redash:status", {"outdated_queries_count": outdated_queries_count, "last_refresh_at": now}) statsd_client.gauge("manager.seconds_since_refresh", now - float(status.get("last_refresh_at", now)))
def refresh(self): self.executions = redis_connection.hgetall(self.KEY_NAME)