Example #1
0
def refresh_schemas():
    """
    Refreshes the data sources schemas.
    """
    blacklist = [
        int(ds_id)
        for ds_id in redis_connection.smembers("data_sources:schema:blacklist")
        if ds_id
    ]
    global_start_time = time.time()

    logger.info(u"task=refresh_schemas state=start")

    for ds in models.DataSource.query:
        if ds.paused:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)",
                ds.id,
                ds.pause_reason,
            )
        elif ds.id in blacklist:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=blacklist",
                ds.id)
        elif ds.org.is_disabled:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=org_disabled",
                ds.id)
        else:
            refresh_schema.delay(ds.id)

    logger.info(
        u"task=refresh_schemas state=finish total_runtime=%.2f",
        time.time() - global_start_time,
    )
Example #2
0
def refresh_schemas():
    """
    Refreshes the data sources schemas.
    """
    blacklist = [
        int(ds_id)
        for ds_id in redis_connection.smembers('data_sources:schema:blacklist')
        if ds_id
    ]
    global_start_time = time.time()

    logger.info(u"task=refresh_schemas state=start")

    for ds in models.DataSource.query:
        if ds.paused:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)",
                ds.id, ds.pause_reason)
        elif ds.id in blacklist:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=blacklist",
                ds.id)
        elif ds.org.is_disabled:
            logger.info(
                u"task=refresh_schema state=skip ds_id=%s reason=org_disabled",
                ds.id)
        else:
            refresh_schema.apply_async(args=(ds.id, ),
                                       queue=settings.SCHEMAS_REFRESH_QUEUE)

    logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f",
                time.time() - global_start_time)
Example #3
0
def refresh_schemas():
    """
    Refreshes the data sources schemas.
    """

    blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]

    global_start_time = time.time()

    logger.info(u"task=refresh_schemas state=start")

    for ds in models.DataSource.select():
        if ds.paused:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
        elif ds.id in blacklist:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
        else:
            logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
            start_time = time.time()
            try:
                ds.get_schema(refresh=True)
                logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
            except Exception:
                logger.exception(u"Failed refreshing schema for the data source: %s", ds.name)
                logger.info(u"task=refresh_schema state=failed ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)

    logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
Example #4
0
def refresh_schemas():
    """
    Refreshes the data sources schemas.
    """

    blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]

    global_start_time = time.time()

    logger.info(u"task=refresh_schemas state=start")

    for ds in models.DataSource.query:
        if ds.paused:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
        elif ds.id in blacklist:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
        else:
            logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
            start_time = time.time()
            try:
                ds.get_schema(refresh=True)
                logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
            except Exception:
                logger.exception(u"Failed refreshing schema for the data source: %s", ds.name)
                logger.info(u"task=refresh_schema state=failed ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)

    logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
Example #5
0
def status_api():
    status = {}
    info = redis_connection.info()
    status['redis_used_memory'] = info['used_memory_human']

    status['queries_count'] = models.Query.select().count()
    status['query_results_count'] = models.QueryResult.select().count()
    status['dashboards_count'] = models.Dashboard.select().count()
    status['widgets_count'] = models.Widget.select().count()

    status['workers'] = [redis_connection.hgetall(w)
                         for w in redis_connection.smembers('workers')]

    manager_status = redis_connection.hgetall('manager:status')
    status['manager'] = manager_status
    status['manager']['queue_size'] = redis_connection.zcard('jobs')

    return jsonify(status)
Example #6
0
def refresh_schemas():
    """
    Refreshes the data sources schemas.
    """
    blacklist = [int(ds_id) for ds_id in redis_connection.smembers('data_sources:schema:blacklist') if ds_id]
    global_start_time = time.time()

    logger.info(u"task=refresh_schemas state=start")

    for ds in models.DataSource.query:
        if ds.paused:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=paused(%s)", ds.id, ds.pause_reason)
        elif ds.id in blacklist:
            logger.info(u"task=refresh_schema state=skip ds_id=%s reason=blacklist", ds.id)
        else:
            refresh_schema.apply_async(args=(ds.id,), queue="schemas")

    logger.info(u"task=refresh_schemas state=finish total_runtime=%.2f", time.time() - global_start_time)
Example #7
0
def status_api():
    status = {}
    info = redis_connection.info()
    status['redis_used_memory'] = info['used_memory_human']

    status['queries_count'] = models.Query.select().count()
    status['query_results_count'] = models.QueryResult.select().count()
    status['dashboards_count'] = models.Dashboard.select().count()
    status['widgets_count'] = models.Widget.select().count()

    status['workers'] = [
        redis_connection.hgetall(w)
        for w in redis_connection.smembers('workers')
    ]

    manager_status = redis_connection.hgetall('manager:status')
    status['manager'] = manager_status
    status['manager']['queue_size'] = redis_connection.zcard('jobs')

    return jsonify(status)