def save_multiple_serializable_objects(self, obj_list, id_name):
"""
This method saves a single AP object in ES
:param list obj_list: A list of OptHistoryAction() Objects to save in ES
:param str id_name: A string describing the attributes used to get the object id: "actionId", "generalInfo.apId"
:return bool: Returns True on a successful operation
"""
bulk_data = []
for obj in obj_list:
# Add timestamp if it exists
if hasattr(obj, "timestamp"):
obj.timestamp = current_utc_time()
try:
doc_id = multi_getattr(obj, id_name)
except AttributeError as exc:
raise ESAdapterException(
"Couldn't find object id for object %s, %s, %s" %
(class_name(obj), id_name, exc))
obj_dict = {
"_index": self.index_name_to_save_obj(obj),
"_type": self.doc_type_name_to_save_obj(obj),
"_id": doc_id,
"_source": self._to_JSON(obj)
}
bulk_data.append(obj_dict)
# Bulk save the data in ES
es_response = helpers.bulk(self.es, bulk_data)
return self.parse_bulk_response(es_response, obj_list)
def verify_returned_list_size(self, expected_list, actual_list):
"""
This method verifies that a returned list from ES is the same size as the expected one
:param list expected_list:
:param list actual_list:
"""
# Enter a log entry in case the list size is different
if type(expected_list) == list and type(actual_list) == list:
if len(expected_list) != len(actual_list):
self.logger.warning(
"Got wrong list size from ES in %s, trying to get %s" %
(class_name(self), expected_list))
else:
self.logger.warning(
"Got wrong list param type in %s, with params %s, %s" %
(class_name(self), expected_list, actual_list))
def _match_multiple_values(field, value_list):
"""
Match list for a single value
:param str field: The name of the field in ES
:param list value_list:
:return dict:
"""
# Make sure input is a list
if type(value_list) != list:
raise TypeError("%s in %s didn't received a list: %s" %
(func_name(), class_name(ESQuery), value_list))
return {"terms": {field: value_list}}
def parse_aggregation_response_to_dict(self, es_response):
"""
Parses an ES response coming from a query returning a list of aggregated values
The response is parse to {NAME: VALUE} dictionary
:param dict es_response: Response from ES
:return dict object_dict: {Name: Value}
"""
object_dict = {}
agg_dict = extract_value(es_response, "aggregations")
if not agg_dict:
self.logger.error(
"Error receiving aggregation dictionary from es query in %s, %s"
% (class_name(self), es_response))
return object_dict
out_dict = {}
self.aggregation_parser(out_dict, agg_dict)
return out_dict
def parse_es_aggregation_list(self, es_response, agg_name):
"""
This method parses an ES aggregation into a list of values
:param dictionary es_response: The response from ES
:param string agg_name: The name of the aggregation to make a list
:return list :
"""
try:
item_list_doc = extract_value(es_response, 'aggregations',
agg_name, "buckets")
item_list = []
for items in item_list_doc:
item_list.append(items['key'])
return item_list
except KeyError as exc:
raise ESAdapterException(
"Error while trying to create a list in %s: %s" %
(class_name(self), exc))
def get_param_value_from_object(self, doc_id, param_list):
"""
Get a list of field names and returns a dictionary of {param: [values]}
:param str doc_id: The ID of the document
:param list param_list:
:return dict:
"""
if type(param_list) is not list:
raise ESAdapterException(
"get_param_value_from_object got wrong input list type %s in %s"
% (param_list, class_name(self)))
# Get a list of all the action Serial Numbers for the timestamp
es_response = self.es.search(index=self.index_name_to_get_obj(),
body=query_params_from_object(
doc_id, param_list))
param_dict = {}
for param in param_list:
value = self.parse_es_aggregation_list(es_response, param)
if value:
param_dict[param] = value
return param_dict
def _create_aggregation_field_dict(field,
agg_type,
order=DEFAULT_ORDER,
size=DEFAULT_AGG_SIZE,
interval=None):
"""
Adds an aggregation dictionary to the main aggs_dict
:param str field: The name of the field in ES
:param str agg_type: One of the types of the aggregation, must be of ESAdapter constant
:param dict order:
:param int size: The number of terms to bring in an aggregations
:return dict:
"""
if agg_type == ESQuery.SUM or\
agg_type == ESQuery.AVERAGE or\
agg_type == ESQuery.MAX or\
agg_type == ESQuery.MIN or\
agg_type == ESQuery.STATS:
return {agg_type: {"field": field}}
elif agg_type == ESQuery.CARDINALITY:
return {agg_type: {"field": field, "precision_threshold": 100}}
elif agg_type == ESQuery.TOP_HITS:
if not size:
size = 1
source = {"include": field}
sort = {"@timestamp": {"order": "desc"}}
return {
ESQuery.TOP_HITS: {
"size": size,
"_source": source,
"sort": [sort]
}
}
# show extended statistics,
# std_deviation_bounds provides an interval of plus/minus 1 standard deviations from the mean
elif agg_type == ESQuery.EXTENDED_STATS:
return {ESQuery.EXTENDED_STATS: {"field": field, "sigma": 1}}
elif agg_type == ESQuery.TERMS:
return {
ESQuery.TERMS: {
"field": field,
"size": size,
"order": order
}
}
elif agg_type == ESQuery.HISTOGRAM:
return {
ESQuery.HISTOGRAM: {
"field": field,
"interval": interval,
}
}
else:
raise TypeError("%s received wrong aggregation type %s" %
(class_name(ESQuery), agg_type))