Ejemplo n.º 1
0
def pgo_get_data_and_columns_from_view(host_id, view_name, max_days_to_fetch, idb_latest_timestamp=None):
    dt_now = datetime.now()
    from_timestamp = idb_latest_timestamp
    to_timestamp = dt_now

    if from_timestamp is None:
        from_timestamp = dt_now - timedelta(days=max_days_to_fetch)

    if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
        to_timestamp = from_timestamp + timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME)
    else:
        to_timestamp = to_timestamp - timedelta(seconds=SAFETY_SECONDS_FOR_LATEST_DATA)

    if from_timestamp >= to_timestamp:
        return [], None

    sql = open(os.path.join(TEMPLATES_FOLDER, view_name)).read()
    sql_params = {'host_id': host_id, 'from_timestamp': from_timestamp, 'to_timestamp': to_timestamp}

    logging.debug("Executing:")
    logging.debug("%s", datadb.mogrify(sql, sql_params))

    view_data, columns = datadb.execute(sql, sql_params)

    # removing timestamp, we only want to store the utc epoch "time" column
    timestamp_index = columns.index('timestamp')
    if timestamp_index != 0:
        raise Exception('"timestamp" needs to be the 1st column returned!')
    columns.remove('timestamp')

    ret_data = []
    for d in view_data:
        ret_data.append(list(d[1:]))

    return ret_data, columns
Ejemplo n.º 2
0
def pgo_get_data_and_columns_from_view(host_id, view_name, max_days_to_fetch, idb_latest_timestamp=None):
    dt_now = datetime.now()
    from_timestamp = idb_latest_timestamp
    to_timestamp = dt_now

    if from_timestamp is None:
        from_timestamp = dt_now - timedelta(days=max_days_to_fetch)

    if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
        to_timestamp = from_timestamp + timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME)
    else:
        to_timestamp = to_timestamp - timedelta(seconds=SAFETY_SECONDS_FOR_LATEST_DATA)

    if from_timestamp >= to_timestamp:
        return [], None

    sql = open(os.path.join(TEMPLATES_FOLDER, view_name + ".sql")).read()
    sql_params = {"host_id": host_id, "from_timestamp": from_timestamp, "to_timestamp": to_timestamp}

    logging.debug("Executing:")
    logging.debug("%s", datadb.mogrify(sql, sql_params))

    view_data, columns = datadb.executeAsDict(sql, sql_params)

    return view_data, columns
Ejemplo n.º 3
0
def pgo_get_data_and_columns_from_view(host_id, ui_shortname, view_name, max_days_to_fetch, idb_latest_timestamp=None):
    dt_now = datetime.now()
    from_timestamp = idb_latest_timestamp
    to_timestamp = dt_now

    if from_timestamp is None:
        from_timestamp = dt_now - timedelta(days=max_days_to_fetch)

    if from_timestamp < dt_now - timedelta(days=settings.get('max_days_to_select_at_a_time', 3)):
        to_timestamp = from_timestamp + timedelta(days=settings.get('max_days_to_select_at_a_time', 3))
    else:
        to_timestamp = to_timestamp - timedelta(seconds=settings.get('safety_seconds_for_latest_data', 10))

    if from_timestamp >= to_timestamp:
        return [], None

    sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
    sql_params = {'host_id': host_id, 'from_timestamp': from_timestamp, 'to_timestamp': to_timestamp,
                  'lag_interval': settings.get('lag_interval', '4 hours')}

    logging.debug("Executing:")
    logging.debug("%s", datadb.mogrify(sql, sql_params))

    try:
        view_data, columns = datadb.executeAsDict(sql, sql_params)
        return view_data, columns
    except Exception as e:
        logging.error('[%s] could not get data from PGO view %s: %s', ui_shortname, view_name, e)

    return [], []
Ejemplo n.º 4
0
def pgo_get_data_and_columns_from_view(host_id,
                                       view_name,
                                       max_days_to_fetch,
                                       idb_latest_timestamp=None):
    dt_now = datetime.now()
    from_timestamp = idb_latest_timestamp
    to_timestamp = dt_now

    if from_timestamp is None:
        from_timestamp = dt_now - timedelta(days=max_days_to_fetch)

    if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
        to_timestamp = from_timestamp + timedelta(
            days=MAX_DAYS_TO_SELECT_AT_A_TIME)
    else:
        to_timestamp = to_timestamp - timedelta(
            seconds=SAFETY_SECONDS_FOR_LATEST_DATA)

    if from_timestamp >= to_timestamp:
        return [], None

    sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
    sql_params = {
        'host_id': host_id,
        'from_timestamp': from_timestamp,
        'to_timestamp': to_timestamp
    }

    logging.debug("Executing:")
    logging.debug("%s", datadb.mogrify(sql, sql_params))

    view_data, columns = datadb.executeAsDict(sql, sql_params)

    return view_data, columns
Ejemplo n.º 5
0
def pgo_get_data_and_columns_from_view(host_id,
                                       ui_shortname,
                                       view_name,
                                       max_days_to_fetch,
                                       idb_latest_timestamp=None):
    dt_now = datetime.now()
    from_timestamp = idb_latest_timestamp
    to_timestamp = dt_now

    if from_timestamp is None:
        from_timestamp = dt_now - timedelta(days=max_days_to_fetch)

    if from_timestamp < dt_now - timedelta(
            days=settings.get('max_days_to_select_at_a_time', 3)):
        to_timestamp = from_timestamp + timedelta(
            days=settings.get('max_days_to_select_at_a_time', 3))
    else:
        to_timestamp = to_timestamp - timedelta(
            seconds=settings.get('safety_seconds_for_latest_data', 10))

    if from_timestamp >= to_timestamp:
        return [], None

    sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
    sql_params = {
        'host_id': host_id,
        'from_timestamp': from_timestamp,
        'to_timestamp': to_timestamp,
        'lag_interval': settings.get('lag_interval', '4 hours')
    }

    logging.debug("Executing:")
    logging.debug("%s", datadb.mogrify(sql, sql_params))

    try:
        view_data, columns = datadb.executeAsDict(sql, sql_params)
        return view_data, columns
    except Exception as e:
        logging.error('[%s] could not get data from PGO view %s: %s',
                      ui_shortname, view_name, e)

    return [], []