def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task(query.query, query.data_source, scheduled=True, metadata={ 'Query ID': query.id, 'Username': '******' }) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") enqueued = [] for query in models.Query.outdated_queries(): if not _should_refresh_query(query): continue try: enqueue_query( _apply_default_parameters(query), query.data_source, query.user_id, scheduled_query=query, metadata={ "Query ID": query.id, "Username": "******" }, ) enqueued.append(query) except Exception as e: message = "Could not enqueue query %d due to %s" % (query.id, repr(e)) logging.info(message) error = RefreshQueriesError(message).with_traceback( e.__traceback__) sentry.capture_exception(error) status = { "outdated_queries_count": len(enqueued), "last_refresh_at": time.time(), "query_ids": json_dumps([q.id for q in enqueued]), } redis_connection.hmset("redash:status", status) logger.info("Done refreshing queries: %s" % status)
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): # TODO: this should go into lower priority QueryTask.add_task(query.query, query.data_source, scheduled=True) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) # TODO: decide if we still need this # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs')) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: enqueue_query(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): enqueue_query(query.query, query.data_source, scheduled=True, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: enqueue_query(query.query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json.dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): # TODO: this should go into lower priority QueryTask.add_task(query.query, query.data_source, scheduled=True) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) # TODO: decide if we still need this # statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs')) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.debug("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.debug("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: query_text = query.query_text parameters = {p['name']: p.get('value') for p in query.parameters} if any(parameters): try: query_text = query.parameterized.apply(parameters).query except InvalidParameterError as e: error = u"Skipping refresh of {} because of invalid parameters: {}".format(query.id, e.message) track_failure(query, error) continue except QueryDetachedFromDataSourceError as e: error = ("Skipping refresh of {} because a related dropdown " "query ({}) is unattached to any datasource.").format(query.id, e.query_id) track_failure(query, error) continue enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): logger.info("Refreshing queries...") outdated_queries_count = 0 query_ids = [] with statsd_client.timer('manager.outdated_queries_lookup'): for query in models.Query.outdated_queries(): if settings.FEATURE_DISABLE_REFRESH_QUERIES: logging.info("Disabled refresh queries.") elif query.org.is_disabled: logging.debug("Skipping refresh of %s because org is disabled.", query.id) elif query.data_source is None: logging.info("Skipping refresh of %s because the datasource is none.", query.id) elif query.data_source.paused: logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason) else: if query.options and len(query.options.get('parameters', [])) > 0: query_params = {p['name']: p.get('value') for p in query.options['parameters']} query_text = mustache_render(query.query_text, query_params) else: query_text = query.query_text if is_enqueued(query_text, query.data_source.id): logging.info("Skipping refresh of %s because query is already queued up.", query.id) else: enqueue_query(query_text, query.data_source, query.user_id, scheduled_query=query, metadata={'Query ID': query.id, 'Username': '******'}) query_ids.append(query.id) outdated_queries_count += 1 statsd_client.gauge('manager.outdated_queries', outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids)) status = redis_connection.hgetall('redash:status') now = time.time() redis_connection.hmset('redash:status', { 'outdated_queries_count': outdated_queries_count, 'last_refresh_at': now, 'query_ids': json_dumps(query_ids) }) statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries(): # self.status['last_refresh_at'] = time.time() # self._save_status() logger.info("Refreshing queries...") outdated_queries_count = 0 for query in models.Query.outdated_queries(): QueryTask.add_task( query.query, query.data_source, scheduled=True, metadata={"Query ID": query.id, "Username": "******"} ) outdated_queries_count += 1 statsd_client.gauge("manager.outdated_queries", outdated_queries_count) logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count) status = redis_connection.hgetall("redash:status") now = time.time() redis_connection.hmset("redash:status", {"outdated_queries_count": outdated_queries_count, "last_refresh_at": now}) statsd_client.gauge("manager.seconds_since_refresh", now - float(status.get("last_refresh_at", now)))
def update(self, query_id): redis_connection.hmset(self.KEY_NAME, {query_id: time.time()})
def update(self, query_id): redis_connection.hmset(self.KEY_NAME, { query_id: time.time() })