def refresh_queries():
# self.status['last_refresh_at'] = time.time()
# self._save_status()
logger.info("Refreshing queries...")
outdated_queries_count = 0
for query in models.Query.outdated_queries():
QueryTask.add_task(query.query, query.data_source, scheduled=True,
metadata={'Query ID': query.id, 'Username': '******'})
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
# self.status['last_refresh_at'] = time.time()
# self._save_status()
logger.info("Refreshing queries...")
outdated_queries_count = 0
for query in models.Query.outdated_queries():
QueryTask.add_task(query.query,
query.data_source,
scheduled=True,
metadata={
'Query ID': query.id,
'Username': '******'
})
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries." %
outdated_queries_count)
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now
})
statsd_client.gauge('manager.seconds_since_refresh',
now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
enqueued = []
for query in models.Query.outdated_queries():
if not _should_refresh_query(query):
continue
try:
enqueue_query(
_apply_default_parameters(query),
query.data_source,
query.user_id,
scheduled_query=query,
metadata={
"Query ID": query.id,
"Username": "******"
},
)
enqueued.append(query)
except Exception as e:
message = "Could not enqueue query %d due to %s" % (query.id,
repr(e))
logging.info(message)
error = RefreshQueriesError(message).with_traceback(
e.__traceback__)
sentry.capture_exception(error)
status = {
"outdated_queries_count": len(enqueued),
"last_refresh_at": time.time(),
"query_ids": json_dumps([q.id for q in enqueued]),
}
redis_connection.hmset("redash:status", status)
logger.info("Done refreshing queries: %s" % status)
def refresh_queries():
# self.status['last_refresh_at'] = time.time()
# self._save_status()
logger.info("Refreshing queries...")
outdated_queries_count = 0
for query in models.Query.outdated_queries():
# TODO: this should go into lower priority
QueryTask.add_task(query.query, query.data_source, scheduled=True)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
# TODO: decide if we still need this
# statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if query.data_source.paused:
logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason)
else:
enqueue_query(query.query, query.data_source,
scheduled=True,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json.dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
enqueue_query(query.query, query.data_source,
scheduled=True,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json.dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if settings.FEATURE_DISABLE_REFRESH_QUERIES:
logging.info("Disabled refresh queries.")
elif query.data_source.paused:
logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason)
else:
enqueue_query(query.query_text, query.data_source, query.user_id,
scheduled_query=query,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json.dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
# self.status['last_refresh_at'] = time.time()
# self._save_status()
logger.info("Refreshing queries...")
outdated_queries_count = 0
for query in models.Query.outdated_queries():
# TODO: this should go into lower priority
QueryTask.add_task(query.query, query.data_source, scheduled=True)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
# TODO: decide if we still need this
# statsd_client.gauge('manager.queue_size', self.redis_connection.zcard('jobs'))
logger.info("Done refreshing queries. Found %d outdated queries." %
outdated_queries_count)
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now
})
statsd_client.gauge('manager.seconds_since_refresh',
now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if settings.FEATURE_DISABLE_REFRESH_QUERIES:
logging.info("Disabled refresh queries.")
elif query.org.is_disabled:
logging.debug("Skipping refresh of %s because org is disabled.", query.id)
elif query.data_source is None:
logging.debug("Skipping refresh of %s because the datasource is none.", query.id)
elif query.data_source.paused:
logging.debug("Skipping refresh of %s because datasource - %s is paused (%s).",
query.id, query.data_source.name, query.data_source.pause_reason)
else:
query_text = query.query_text
parameters = {p['name']: p.get('value') for p in query.parameters}
if any(parameters):
try:
query_text = query.parameterized.apply(parameters).query
except InvalidParameterError as e:
error = u"Skipping refresh of {} because of invalid parameters: {}".format(query.id, e.message)
track_failure(query, error)
continue
except QueryDetachedFromDataSourceError as e:
error = ("Skipping refresh of {} because a related dropdown "
"query ({}) is unattached to any datasource.").format(query.id, e.query_id)
track_failure(query, error)
continue
enqueue_query(query_text, query.data_source, query.user_id,
scheduled_query=query,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json_dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
logger.info("Refreshing queries...")
outdated_queries_count = 0
query_ids = []
with statsd_client.timer('manager.outdated_queries_lookup'):
for query in models.Query.outdated_queries():
if settings.FEATURE_DISABLE_REFRESH_QUERIES:
logging.info("Disabled refresh queries.")
elif query.org.is_disabled:
logging.debug("Skipping refresh of %s because org is disabled.", query.id)
elif query.data_source is None:
logging.info("Skipping refresh of %s because the datasource is none.", query.id)
elif query.data_source.paused:
logging.info("Skipping refresh of %s because datasource - %s is paused (%s).", query.id, query.data_source.name, query.data_source.pause_reason)
else:
if query.options and len(query.options.get('parameters', [])) > 0:
query_params = {p['name']: p.get('value')
for p in query.options['parameters']}
query_text = mustache_render(query.query_text, query_params)
else:
query_text = query.query_text
if is_enqueued(query_text, query.data_source.id):
logging.info("Skipping refresh of %s because query is already queued up.", query.id)
else:
enqueue_query(query_text, query.data_source, query.user_id,
scheduled_query=query,
metadata={'Query ID': query.id, 'Username': '******'})
query_ids.append(query.id)
outdated_queries_count += 1
statsd_client.gauge('manager.outdated_queries', outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries: %s" % (outdated_queries_count, query_ids))
status = redis_connection.hgetall('redash:status')
now = time.time()
redis_connection.hmset('redash:status', {
'outdated_queries_count': outdated_queries_count,
'last_refresh_at': now,
'query_ids': json_dumps(query_ids)
})
statsd_client.gauge('manager.seconds_since_refresh', now - float(status.get('last_refresh_at', now)))
def refresh_queries():
# self.status['last_refresh_at'] = time.time()
# self._save_status()
logger.info("Refreshing queries...")
outdated_queries_count = 0
for query in models.Query.outdated_queries():
QueryTask.add_task(
query.query, query.data_source, scheduled=True, metadata={"Query ID": query.id, "Username": "******"}
)
outdated_queries_count += 1
statsd_client.gauge("manager.outdated_queries", outdated_queries_count)
logger.info("Done refreshing queries. Found %d outdated queries." % outdated_queries_count)
status = redis_connection.hgetall("redash:status")
now = time.time()
redis_connection.hmset("redash:status", {"outdated_queries_count": outdated_queries_count, "last_refresh_at": now})
statsd_client.gauge("manager.seconds_since_refresh", now - float(status.get("last_refresh_at", now)))
def update(self, query_id):
redis_connection.hmset(self.KEY_NAME, {query_id: time.time()})
def update(self, query_id):
redis_connection.hmset(self.KEY_NAME, {
query_id: time.time()
})
