def test_fetch_query(es):
"""Test executing fetch query.
Notes:
if is_fetch is ticked, this function checks if the FETCH_QUERY returns results.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
Returns:
(dict).The results of the query if they are returned.
"""
query = QueryString(query=str(TIME_FIELD) + ":* AND " + FETCH_QUERY)
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
if total_results > 0:
return response
else:
# failed to get the TIME_FIELD with the FETCH_QUERY
# this can happen and not be an error if the FETCH_QUERY doesn't have results yet.
# Thus this does not return an error message
return None
def test_time_field_query(es):
"""Test executing query of fetch time field.
Notes:
if is_fetch is ticked, this function checks if the entered TIME_FIELD returns results.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
Returns:
(dict).The results of the query if they are returned.
"""
query = QueryString(query=TIME_FIELD + ':*')
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
if total_results == 0:
# failed in getting the TIME_FIELD
return_error(
"Fetch incidents test failed.\nDate field value incorrect [{}].".
format(TIME_FIELD))
else:
return response
def fetch_incidents(proxies):
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch, _ = parse_date_range(date_range=FETCH_TIME,
date_format='%Y-%m-%dT%H:%M:%S.%f',
utc=False,
to_timestamp=False)
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if timestamp: get the last fetch to the correct format of timestamp
if 'Timestamp' in TIME_METHOD:
last_fetch = get_timestamp_first_fetch(last_fetch)
last_fetch_timestamp = last_fetch
# if method is simple date - convert the date string to datetime
elif 'Simple-Date' == TIME_METHOD:
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if last_fetch is set and we are in a "Timestamp" method - than the last_fetch_timestamp is the last_fetch.
else:
last_fetch_timestamp = last_fetch
es = elasticsearch_builder(proxies)
query = QueryString(query=FETCH_QUERY + " AND " + TIME_FIELD + ":*")
# Elastic search can use epoch timestamps (in milliseconds) as date representation regardless of date format.
search = Search(using=es, index=FETCH_INDEX).filter(
{'range': {
TIME_FIELD: {
'gt': last_fetch_timestamp
}
}})
search = search.sort({TIME_FIELD: {
'order': 'asc'
}})[0:FETCH_SIZE].query(query)
response = search.execute().to_dict()
_, total_results = get_total_results(response)
incidents = [] # type: List
if total_results > 0:
if 'Timestamp' in TIME_METHOD:
incidents, last_fetch = results_to_incidents_timestamp(
response, last_fetch)
demisto.setLastRun({'time': last_fetch})
else:
incidents, last_fetch = results_to_incidents_datetime(
response, last_fetch)
demisto.setLastRun({'time': str(last_fetch)})
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.incidents(incidents)
def get_basic_search_query(
entity,
term,
permission_filters_by_entity=None,
offset=0,
limit=100,
fields_to_exclude=None,
fuzzy=False,
):
"""
Performs basic search for the given term in the given entity using the SEARCH_FIELDS.
It also returns number of results in other entities.
:param permission_filters_by_entity: List of pairs of entities and corresponding permission
filters. Only entities in this list are included in the
results, and those are entities are also filtered using
the corresponding permission filters.
"""
limit = _clip_limit(offset, limit)
search_apps = tuple(get_global_search_apps_as_mapping().values())
indices = [app.search_model.get_read_alias() for app in search_apps]
fields = set(chain.from_iterable(app.search_model.SEARCH_FIELDS for app in search_apps))
# Sort the fields so that this function is deterministic
# and the same query is always generated with the same inputs
fields = sorted(fields)
query = _build_term_query(term, fields=fields, fuzzy=fuzzy)
search = Search(index=indices).query(query)
permission_query = _build_global_permission_query(permission_filters_by_entity)
if permission_query:
search = search.filter(permission_query)
search = search.post_filter(
Bool(
should=Term(_document_type=entity.get_app_name()),
),
).sort(
'_score',
'id',
).source(
excludes=fields_to_exclude,
).extra(
track_total_hits=True,
)
search.aggs.bucket(
'count_by_type', 'terms', field='_document_type',
)
return search[offset:offset + limit]
def make_opensearch(index,
filters,
queries=None,
exclusion_filters=None,
range_filters=None,
prefix_filters=None,
terms_filters=None,
es_url='https://opensearch.lco.global'):
"""
Make an OpenSearch query
Parameters
----------
index : str
Name of index to search
filters : list of dicts
Each dict has a criterion for an OpenSearch "filter"
queries : list of dicts
Each dict has a "type" and "query" entry. The 'query' entry is a dict that has a criterion for an
OpenSearch "query"
exclusion_filters : list of dicts
Each dict has a criterion for an OpenSearch "exclude"
range_filters: list of dicts
Each dict has a criterion an OpenSearch "range filter"
prefix_filters:
terms_filters:
es_url : str
URL of the OpenSearch host
Returns
-------
search : opensearch_dsl.Search
The OpenSearch object
"""
if queries is None:
queries = []
if exclusion_filters is None:
exclusion_filters = []
if range_filters is None:
range_filters = []
if terms_filters is None:
terms_filters = []
if prefix_filters is None:
prefix_filters = []
es = OpenSearch(es_url)
s = Search(using=es, index=index)
for f in filters:
s = s.filter('term', **f)
for f in terms_filters:
s = s.filter('terms', **f)
for f in range_filters:
s = s.filter('range', **f)
for f in prefix_filters:
s = s.filter('prefix', **f)
for f in exclusion_filters:
s = s.exclude('term', **f)
for q in queries:
s = s.query(q['type'], **q['query'])
return s
def test_general_query(es):
"""Test executing query to all available indexes.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
"""
try:
query = QueryString(query='*')
search = Search(using=es, index='*').query(query)[0:1]
response = search.execute().to_dict()
get_total_results(response)
except NotFoundError as e:
return_error("Failed executing general search command - please check the Server URL and port number "
"and the supplied credentials.\nError message: {}.".format(str(e)))
def get_search_by_entities_query(
entities,
term=None,
filter_data=None,
composite_field_mapping=None,
permission_filters=None,
ordering=None,
fields_to_include=None,
fields_to_exclude=None,
):
"""
Performs filtered search for the given term across given entities.
"""
filter_data = filter_data or {}
query = []
if term != '':
for entity in entities:
query.append(_build_term_query(term, fields=entity.SEARCH_FIELDS))
filters, ranges = _split_range_fields(filter_data)
# document must match all filters in the list (and)
must_filter = _build_must_queries(filters, ranges, composite_field_mapping)
s = Search(
index=[
entity.get_read_alias()
for entity in entities
],
).query(
Bool(must=query),
).extra(
track_total_hits=True,
)
permission_query = _build_entity_permission_query(permission_filters)
if permission_query:
s = s.filter(permission_query)
s = s.filter(Bool(must=must_filter))
s = _apply_sorting_to_query(s, ordering)
return _apply_source_filtering_to_query(
s,
fields_to_include=fields_to_include,
fields_to_exclude=fields_to_exclude,
)
def build_search(es_connection, index, query, sort='@timestamp', limit_to_fields=[]):
"""[summary]
Args:
index ([string]): [Index pattern to search against]
query ([string]): [Lucene query to limit results]
sort (str, optional): [Sort filter]. Defaults to '@timestamp'.
limit_to_fields (list, optional): [Limit which fields to return]. Defaults to [].
Returns:
[type]: [description]
"""
search = Search(using=es_connection, index=index, doc_type='_doc')
search = search.query('query_string', query=query)
if len(limit_to_fields) != 0:
search = search.source(limit_to_fields)
search = search.sort(sort)
return search
def test_query_to_fetch_incident_index(es):
"""Test executing query in fetch index.
Notes:
if is_fetch it ticked, this function runs a generay query to Elasticsearch just to make sure we get a response
from the FETCH_INDEX.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
"""
try:
query = QueryString(query='*')
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
except NotFoundError as e:
return_error("Fetch incidents test failed.\nError message: {}.".format(str(e).split(',')[2][2:-1]))
def multiple_aggregate_search(es_connection, index_pattern, search_query, aggregation_type, aggregation_field_one, aggregation_field_two, sort='@timestamp', limit_to_fields=[], date_start='now-1d/d', date_end='now'):
s = Search(using=es_connection, index=index_pattern, doc_type='_doc')
s = s.query('query_string', query=search_query)
if len(limit_to_fields) != 0:
s = s.source(limit_to_fields)
s = s.sort(sort)
s = s.filter('range', **{'@timestamp': {'gte': date_start, 'lt': date_end}})
# The four lines above could be summarized into the line below based on your preference:
# s = Search(using=es_connection, index='lab4.1-complete', doc_type='_doc').query('query_string', query='tags:internal_source').source(['source_ip']).sort('source_ip')
s.aggs.bucket(aggregation_field_one, 'terms', field=aggregation_field_one, size=999999).metric('Count', aggregation_type, field=aggregation_field_one)
s.aggs.bucket(aggregation_field_two, 'terms', field=aggregation_field_two, size=999999).metric('Count', aggregation_type, field=aggregation_field_two)
response = s.execute()
aggregation_one = [ x['key'] for x in response.aggregations[aggregation_field_one].buckets ]
aggregation_two = [ x['key'] for x in response.aggregations[aggregation_field_two].buckets ]
return { aggregation_one[i]: aggregation_two[i] for i in range(len(aggregation_one)) }
return list(zip([ x['key'] for x in response.aggregations[aggregation_field_one].buckets ], [ x['key'] for x in response.aggregations[aggregation_field_two].buckets ]))
def search_command(proxies):
"""Performs a search in Elasticsearch."""
index = demisto.args().get('index')
query = demisto.args().get('query')
fields = demisto.args().get('fields') # fields to display
explain = 'true' == demisto.args().get('explain')
base_page = int(demisto.args().get('page'))
size = int(demisto.args().get('size'))
sort_field = demisto.args().get('sort-field')
sort_order = demisto.args().get('sort-order')
es = elasticsearch_builder(proxies)
que = QueryString(query=query)
search = Search(using=es,
index=index).query(que)[base_page:base_page + size]
if explain:
# if 'explain parameter is set to 'true' - adds explanation section to search results
search = search.extra(explain=True)
if fields is not None:
fields = fields.split(',')
search = search.source(fields)
if sort_field is not None:
search = search.sort({sort_field: {'order': sort_order}})
response = search.execute().to_dict()
total_dict, total_results = get_total_results(response)
search_context, meta_headers, hit_tables, hit_headers = results_to_context(
index, query, base_page, size, total_dict, response)
search_human_readable = tableToMarkdown('Search Metadata:',
search_context,
meta_headers,
removeNull=True)
hits_human_readable = tableToMarkdown('Hits:',
hit_tables,
hit_headers,
removeNull=True)
total_human_readable = search_human_readable + '\n' + hits_human_readable
full_context = {
'Elasticsearch.Search(val.Query == obj.Query && val.Index == obj.Index '
'&& val.Server == obj.Server && val.Page == obj.Page && val.Size == obj.Size)':
search_context
}
return_outputs(total_human_readable, full_context, response)
def aggregate_search(es_connection, index_pattern, search_query, aggregation_type, aggregation_field, sort='@timestamp', limit_to_fields=[], date_start='now-1d/d', date_end='now', result_size=100, interval='auto'):
s = Search(using=es_connection, index=index_pattern, doc_type='_doc')
s = s.query('query_string', query=search_query)
if len(limit_to_fields) != 0:
s = s.source(limit_to_fields)
s = s.sort(sort)
if date_start != 'ignore':
s = s.filter('range', **{sort: {'gte': date_start, 'lt': date_end}})
s.aggs.bucket(aggregation_field, 'terms', field=aggregation_field, size=result_size)
if aggregation_type == 'date_histogram':
s.aggs[aggregation_field].metric('Count', aggregation_type, field=aggregation_field, interval=interval)
else:
s.aggs[aggregation_field].metric('Count', aggregation_type, field=aggregation_field)
response = s.execute()
if aggregation_type in ["terms", "auto_date_histogram", "date_histogram"]:
data = [ x for x in response.aggregations[aggregation_field].buckets ]
return_dict = {}
for row in data:
field = row['key']
value = row['doc_count']
return_dict[field] = value
return return_dict
else:
return [ x for x in response.aggregations[aggregation_field].buckets ]
def _Search(self, indexname):
"""
it returns the object which can be used for retreiving certain value from the DB
"""
return Search(using=self.client, index=indexname)