def pgo_get_data_and_columns_from_view(host_id, view_name, max_days_to_fetch, idb_latest_timestamp=None):
dt_now = datetime.now()
from_timestamp = idb_latest_timestamp
to_timestamp = dt_now
if from_timestamp is None:
from_timestamp = dt_now - timedelta(days=max_days_to_fetch)
if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
to_timestamp = from_timestamp + timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME)
else:
to_timestamp = to_timestamp - timedelta(seconds=SAFETY_SECONDS_FOR_LATEST_DATA)
if from_timestamp >= to_timestamp:
return [], None
sql = open(os.path.join(TEMPLATES_FOLDER, view_name)).read()
sql_params = {'host_id': host_id, 'from_timestamp': from_timestamp, 'to_timestamp': to_timestamp}
logging.debug("Executing:")
logging.debug("%s", datadb.mogrify(sql, sql_params))
view_data, columns = datadb.execute(sql, sql_params)
# removing timestamp, we only want to store the utc epoch "time" column
timestamp_index = columns.index('timestamp')
if timestamp_index != 0:
raise Exception('"timestamp" needs to be the 1st column returned!')
columns.remove('timestamp')
ret_data = []
for d in view_data:
ret_data.append(list(d[1:]))
return ret_data, columns
def pgo_get_data_and_columns_from_view(host_id, view_name, max_days_to_fetch, idb_latest_timestamp=None):
dt_now = datetime.now()
from_timestamp = idb_latest_timestamp
to_timestamp = dt_now
if from_timestamp is None:
from_timestamp = dt_now - timedelta(days=max_days_to_fetch)
if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
to_timestamp = from_timestamp + timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME)
else:
to_timestamp = to_timestamp - timedelta(seconds=SAFETY_SECONDS_FOR_LATEST_DATA)
if from_timestamp >= to_timestamp:
return [], None
sql = open(os.path.join(TEMPLATES_FOLDER, view_name + ".sql")).read()
sql_params = {"host_id": host_id, "from_timestamp": from_timestamp, "to_timestamp": to_timestamp}
logging.debug("Executing:")
logging.debug("%s", datadb.mogrify(sql, sql_params))
view_data, columns = datadb.executeAsDict(sql, sql_params)
return view_data, columns
def pgo_get_data_and_columns_from_view(host_id, ui_shortname, view_name, max_days_to_fetch, idb_latest_timestamp=None):
dt_now = datetime.now()
from_timestamp = idb_latest_timestamp
to_timestamp = dt_now
if from_timestamp is None:
from_timestamp = dt_now - timedelta(days=max_days_to_fetch)
if from_timestamp < dt_now - timedelta(days=settings.get('max_days_to_select_at_a_time', 3)):
to_timestamp = from_timestamp + timedelta(days=settings.get('max_days_to_select_at_a_time', 3))
else:
to_timestamp = to_timestamp - timedelta(seconds=settings.get('safety_seconds_for_latest_data', 10))
if from_timestamp >= to_timestamp:
return [], None
sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
sql_params = {'host_id': host_id, 'from_timestamp': from_timestamp, 'to_timestamp': to_timestamp,
'lag_interval': settings.get('lag_interval', '4 hours')}
logging.debug("Executing:")
logging.debug("%s", datadb.mogrify(sql, sql_params))
try:
view_data, columns = datadb.executeAsDict(sql, sql_params)
return view_data, columns
except Exception as e:
logging.error('[%s] could not get data from PGO view %s: %s', ui_shortname, view_name, e)
return [], []
def pgo_get_data_and_columns_from_view(host_id,
view_name,
max_days_to_fetch,
idb_latest_timestamp=None):
dt_now = datetime.now()
from_timestamp = idb_latest_timestamp
to_timestamp = dt_now
if from_timestamp is None:
from_timestamp = dt_now - timedelta(days=max_days_to_fetch)
if from_timestamp < dt_now - timedelta(days=MAX_DAYS_TO_SELECT_AT_A_TIME):
to_timestamp = from_timestamp + timedelta(
days=MAX_DAYS_TO_SELECT_AT_A_TIME)
else:
to_timestamp = to_timestamp - timedelta(
seconds=SAFETY_SECONDS_FOR_LATEST_DATA)
if from_timestamp >= to_timestamp:
return [], None
sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
sql_params = {
'host_id': host_id,
'from_timestamp': from_timestamp,
'to_timestamp': to_timestamp
}
logging.debug("Executing:")
logging.debug("%s", datadb.mogrify(sql, sql_params))
view_data, columns = datadb.executeAsDict(sql, sql_params)
return view_data, columns
def pgo_get_data_and_columns_from_view(host_id,
ui_shortname,
view_name,
max_days_to_fetch,
idb_latest_timestamp=None):
dt_now = datetime.now()
from_timestamp = idb_latest_timestamp
to_timestamp = dt_now
if from_timestamp is None:
from_timestamp = dt_now - timedelta(days=max_days_to_fetch)
if from_timestamp < dt_now - timedelta(
days=settings.get('max_days_to_select_at_a_time', 3)):
to_timestamp = from_timestamp + timedelta(
days=settings.get('max_days_to_select_at_a_time', 3))
else:
to_timestamp = to_timestamp - timedelta(
seconds=settings.get('safety_seconds_for_latest_data', 10))
if from_timestamp >= to_timestamp:
return [], None
sql = open(os.path.join(TEMPLATES_FOLDER, view_name + '.sql')).read()
sql_params = {
'host_id': host_id,
'from_timestamp': from_timestamp,
'to_timestamp': to_timestamp,
'lag_interval': settings.get('lag_interval', '4 hours')
}
logging.debug("Executing:")
logging.debug("%s", datadb.mogrify(sql, sql_params))
try:
view_data, columns = datadb.executeAsDict(sql, sql_params)
return view_data, columns
except Exception as e:
logging.error('[%s] could not get data from PGO view %s: %s',
ui_shortname, view_name, e)
return [], []