def get_number_series_for_query(backend_host, backend_port, user, password,
schema, query):
nb_series_query = influx_query_modification.add_limit(query, 1)
result_df_dict = influx_querying.pd_query(backend_host, backend_port, user,
password, schema,
nb_series_query)
return influx_querying.get_nb_series_in_pd_result(result_df_dict)
def get_data(self, user, password, schema, query):
# prevent expensive parsing of query case not select
if not query.upper().startswith('SELECT '):
return None
parsed_query = influx_query_parsing.sqlparse_query(query)
if not influx_query_parsing.is_select(parsed_query):
return None
query_is_modified = False
from_parts = influx_query_parsing.extract_measurement_from_query(schema, parsed_query)
query_auto_rp = influx_rp_auto_selection.update_query_with_right_rp(from_parts, query, parsed_query,
self.retention_policies,
self.aggregation_properties, False)
if query_auto_rp is not None:
query_is_modified = True
query = query_auto_rp
if self.max_nb_points_per_query is not None:
query_limit_nb_points = influx_rp_auto_selection.update_query_to_limit_nb_points_for_query(
self.backend_host, self.backend_port, user, password, from_parts,
query, parsed_query,
self.aggregation_properties,
self.max_nb_points_per_query)
if query_limit_nb_points is not None:
query_is_modified = True
query = query_limit_nb_points
elif self.max_nb_points_per_series is not None:
query_limit_nb_points = influx_rp_auto_selection.update_query_to_limit_nb_points_per_series(
from_parts, query, parsed_query,
self.aggregation_properties, self.max_nb_points_per_series)
if query_limit_nb_points is not None:
query_is_modified = True
query = query_limit_nb_points
# for rule in self.rules.itervalues():
if 'handle_counter_wrap_non_negative_derivative' in self.rules:
if from_parts['schema'] is not None and from_parts['schema'] in self.counter_overflows:
measurement_overflows = self.counter_overflows[from_parts['schema']]
if from_parts['measurement'] in measurement_overflows:
if from_parts['measurement'] in measurement_overflows:
# NB: should do it for each field, instead of for whole measurement
rule = self.rules['handle_counter_wrap_non_negative_derivative']
if rule.check(query):
more = {'overflow_value': measurement_overflows[from_parts['measurement']]}
return rule.action(user, password, schema, query, more)
if 'remove_partial_intervals_case_sum_group_by_time' in self.rules:
rule = self.rules['remove_partial_intervals_case_sum_group_by_time']
if rule.check(query, parsed_query):
return rule.action(user, password, schema, query, parsed_query)
if query_is_modified:
result_df_dict = pd_query(self.backend_host, self.backend_port, user, password, schema, query)
return result_df_dict
return None
def action(self, user, password, schema, query, parsed_query, more=None):
query, group_by_interval = self.rework_query(query, parsed_query)
result_df_dict = pd_query(self.backend_host, self.backend_port, user,
password, schema, query)
result_df_dict = self.rework_data(result_df_dict, group_by_interval)
# NB: datetime is the index (df.index), only one column for which values can be retrieved via df.values
return result_df_dict
def action(self, user, password, schema, query, more=None):
overflow_value = more['overflow_value']
nnd_interval_list = influx_query_parsing.extract_non_negative_derivative_time_interval(
query)
nnd_column_list = influx_query_parsing.extract_non_negative_derivative_column_name(
query)
nnd_interval_ms_list = []
for nnd_interval in nnd_interval_list:
nnd_interval_delta = influx_date_manipulation.influx_interval_to_timedelta(
nnd_interval)
nnd_interval_ms = int(nnd_interval_delta.total_seconds() * 1000)
nnd_interval_ms_list.append(nnd_interval_ms)
nb_default_column_name = 0
default_column_name_map = {}
for i, nnd_column in enumerate(nnd_column_list):
if nnd_column == 'non_negative_derivative':
if nb_default_column_name >= 1:
nnd_column_list[i] = nnd_column + '_' + str(i)
nb_default_column_name += 1
default_column_name_map[i] = nnd_column
alt_query = influx_query_modification.remove_non_negative_derivative(
query, None, forced_column_name_map=default_column_name_map)
if alt_query is None:
return None
group_by_interval_influx = influx_query_parsing.extract_time_interval_group_by(
query)
if group_by_interval_influx is None:
logging.error(
'Could not extract group by time interval from query')
return None
group_by_interval_parts = influx_date_manipulation.split_influx_time(
group_by_interval_influx)
number_group_by_interval = group_by_interval_parts['number']
unit_group_by_interval = group_by_interval_parts['unit']
query_time_shift = str(
2 * number_group_by_interval) + unit_group_by_interval
alt_query = influx_query_modification.extend_lower_time_bound(
alt_query, query_time_shift)
result_df_dict = pd_query(self.backend_host, self.backend_port, user,
password, schema, alt_query)
# remove counter wrapping
for series_name in result_df_dict:
df = result_df_dict[series_name]
prev_value = None
for index, row in df.iterrows():
for nnd_column in nnd_column_list:
value = row[nnd_column]
if numpy.isnan(value):
continue
if prev_value is None:
prev_value = value
continue
diff = value - prev_value
if diff < 0:
shift = overflow_value - abs(diff)
while shift <= 0:
shift += overflow_value
new_value = prev_value + shift
df.at[index, nnd_column] = new_value
prev_value = new_value
else:
prev_value = value
result_df_dict[series_name] = df
# apply nnd
for series_name in result_df_dict:
df = result_df_dict[series_name]
prev_value = None
prev_index = None
first_index = None
for index, row in df.iterrows():
for i, nnd_column in enumerate(nnd_column_list):
value = row[nnd_column]
if numpy.isnan(value):
continue
if prev_value is None:
prev_value = value
prev_index = index
first_index = index
df.at[index, nnd_column] = 0
continue
diff = value - prev_value
if diff < 0:
df.at[index, nnd_column] = 0
else:
time_diff = index.value - prev_index.value
new_value = diff * nnd_interval_ms_list[i] / time_diff
df.at[index, nnd_column] = new_value
prev_value = value
prev_index = index
if first_index is not None:
df = df.drop(first_index)
result_df_dict[series_name] = df
return result_df_dict