def refresh_schemas():
"""
Refreshes the data sources schemas.
"""
blacklist = [
int(ds_id)
for ds_id in redis_connection.smembers("data_sources:schema:blacklist")
if ds_id
]
global_start_time = time.time()
logger.info(u"task=refresh_schemas state=start")
for ds in models.DataSource.query:
if ds.paused:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)",
ds.id,
ds.pause_reason,
)
elif ds.id in blacklist:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=blacklist",
ds.id)
elif ds.org.is_disabled:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=org_disabled",
ds.id)
else:
refresh_schema.delay(ds.id)
logger.info(
u"task=refresh_schemas state=finish total_runtime=%.2f",
time.time() - global_start_time,
)
def refresh_schemas():
"""
Refreshes the data sources schemas.
"""
blacklist = [
int(ds_id)
for ds_id in redis_connection.smembers('data_sources:schema:blacklist')
if ds_id
]
global_start_time = time.time()
logger.info(u"task=refresh_schemas state=start")
for ds in models.DataSource.query:
if ds.paused:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)",
ds.id, ds.pause_reason)
elif ds.id in blacklist:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=blacklist",
ds.id)
elif ds.org.is_disabled:
logger.info(
u"task=refresh_schema state=skip ds_id=%s reason=org_disabled",
ds.id)
else:
refresh_schema.apply_async(args=(ds.id, ),
queue=settings.SCHEMAS_REFRESH_QUEUE)
logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f",
time.time() - global_start_time)
def refresh_schemas():
"""
Refreshes the data sources schemas.
"""
blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]
global_start_time = time.time()
logger.info(u"task=refresh_schemas state=start")
for ds in models.DataSource.select():
if ds.paused:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
elif ds.id in blacklist:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
else:
logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
start_time = time.time()
try:
ds.get_schema(refresh=True)
logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
except Exception:
logger.exception(u"Failed refreshing schema for the data source: %s", ds.name)
logger.info(u"task=refresh_schema state=failed ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
def refresh_schemas():
"""
Refreshes the data sources schemas.
"""
blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]
global_start_time = time.time()
logger.info(u"task=refresh_schemas state=start")
for ds in models.DataSource.query:
if ds.paused:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
elif ds.id in blacklist:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
else:
logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
start_time = time.time()
try:
ds.get_schema(refresh=True)
logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
except Exception:
logger.exception(u"Failed refreshing schema for the data source: %s", ds.name)
logger.info(u"task=refresh_schema state=failed ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
def status_api():
status = {}
info = redis_connection.info()
status['redis_used_memory'] = info['used_memory_human']
status['queries_count'] = models.Query.select().count()
status['query_results_count'] = models.QueryResult.select().count()
status['dashboards_count'] = models.Dashboard.select().count()
status['widgets_count'] = models.Widget.select().count()
status['workers'] = [redis_connection.hgetall(w)
for w in redis_connection.smembers('workers')]
manager_status = redis_connection.hgetall('manager:status')
status['manager'] = manager_status
status['manager']['queue_size'] = redis_connection.zcard('jobs')
return jsonify(status)
def refresh_schemas():
"""
Refreshes the data sources schemas.
"""
blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]
global_start_time = time.time()
logger.info(u"task=refresh_schemas state=start")
for ds in models.DataSource.query:
if ds.paused:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
elif ds.id in blacklist:
logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
else:
refresh_schema.apply_async(args=(ds.id,), queue="schemas")
logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
def status_api():
status = {}
info = redis_connection.info()
status['redis_used_memory'] = info['used_memory_human']
status['queries_count'] = models.Query.select().count()
status['query_results_count'] = models.QueryResult.select().count()
status['dashboards_count'] = models.Dashboard.select().count()
status['widgets_count'] = models.Widget.select().count()
status['workers'] = [
redis_connection.hgetall(w)
for w in redis_connection.smembers('workers')
]
manager_status = redis_connection.hgetall('manager:status')
status['manager'] = manager_status
status['manager']['queue_size'] = redis_connection.zcard('jobs')
return jsonify(status)
