def test_fetch_query(es):
"""Test executing fetch query.
Notes:
if is_fetch is ticked, this function checks if the FETCH_QUERY returns results.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
Returns:
(dict).The results of the query if they are returned.
"""
query = QueryString(query=str(TIME_FIELD) + ":* AND " + FETCH_QUERY)
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
if total_results > 0:
return response
else:
# failed to get the TIME_FIELD with the FETCH_QUERY
# this can happen and not be an error if the FETCH_QUERY doesn't have results yet.
# Thus this does not return an error message
return None
def test_time_field_query(es):
"""Test executing query of fetch time field.
Notes:
if is_fetch is ticked, this function checks if the entered TIME_FIELD returns results.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
Returns:
(dict).The results of the query if they are returned.
"""
query = QueryString(query=TIME_FIELD + ':*')
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
if total_results == 0:
# failed in getting the TIME_FIELD
return_error(
"Fetch incidents test failed.\nDate field value incorrect [{}].".
format(TIME_FIELD))
else:
return response
def fetch_incidents(proxies):
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch, _ = parse_date_range(date_range=FETCH_TIME,
date_format='%Y-%m-%dT%H:%M:%S.%f',
utc=False,
to_timestamp=False)
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if timestamp: get the last fetch to the correct format of timestamp
if 'Timestamp' in TIME_METHOD:
last_fetch = get_timestamp_first_fetch(last_fetch)
last_fetch_timestamp = last_fetch
# if method is simple date - convert the date string to datetime
elif 'Simple-Date' == TIME_METHOD:
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if last_fetch is set and we are in a "Timestamp" method - than the last_fetch_timestamp is the last_fetch.
else:
last_fetch_timestamp = last_fetch
es = elasticsearch_builder(proxies)
query = QueryString(query=FETCH_QUERY + " AND " + TIME_FIELD + ":*")
# Elastic search can use epoch timestamps (in milliseconds) as date representation regardless of date format.
search = Search(using=es, index=FETCH_INDEX).filter(
{'range': {
TIME_FIELD: {
'gt': last_fetch_timestamp
}
}})
search = search.sort({TIME_FIELD: {
'order': 'asc'
}})[0:FETCH_SIZE].query(query)
response = search.execute().to_dict()
_, total_results = get_total_results(response)
incidents = [] # type: List
if total_results > 0:
if 'Timestamp' in TIME_METHOD:
incidents, last_fetch = results_to_incidents_timestamp(
response, last_fetch)
demisto.setLastRun({'time': last_fetch})
else:
incidents, last_fetch = results_to_incidents_datetime(
response, last_fetch)
demisto.setLastRun({'time': str(last_fetch)})
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.incidents(incidents)
def test_general_query(es):
"""Test executing query to all available indexes.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
"""
try:
query = QueryString(query='*')
search = Search(using=es, index='*').query(query)[0:1]
response = search.execute().to_dict()
get_total_results(response)
except NotFoundError as e:
return_error("Failed executing general search command - please check the Server URL and port number "
"and the supplied credentials.\nError message: {}.".format(str(e)))
def search_command(proxies):
"""Performs a search in Elasticsearch."""
index = demisto.args().get('index')
query = demisto.args().get('query')
fields = demisto.args().get('fields') # fields to display
explain = 'true' == demisto.args().get('explain')
base_page = int(demisto.args().get('page'))
size = int(demisto.args().get('size'))
sort_field = demisto.args().get('sort-field')
sort_order = demisto.args().get('sort-order')
es = elasticsearch_builder(proxies)
que = QueryString(query=query)
search = Search(using=es,
index=index).query(que)[base_page:base_page + size]
if explain:
# if 'explain parameter is set to 'true' - adds explanation section to search results
search = search.extra(explain=True)
if fields is not None:
fields = fields.split(',')
search = search.source(fields)
if sort_field is not None:
search = search.sort({sort_field: {'order': sort_order}})
response = search.execute().to_dict()
total_dict, total_results = get_total_results(response)
search_context, meta_headers, hit_tables, hit_headers = results_to_context(
index, query, base_page, size, total_dict, response)
search_human_readable = tableToMarkdown('Search Metadata:',
search_context,
meta_headers,
removeNull=True)
hits_human_readable = tableToMarkdown('Hits:',
hit_tables,
hit_headers,
removeNull=True)
total_human_readable = search_human_readable + '\n' + hits_human_readable
full_context = {
'Elasticsearch.Search(val.Query == obj.Query && val.Index == obj.Index '
'&& val.Server == obj.Server && val.Page == obj.Page && val.Size == obj.Size)':
search_context
}
return_outputs(total_human_readable, full_context, response)
def test_query_to_fetch_incident_index(es):
"""Test executing query in fetch index.
Notes:
if is_fetch it ticked, this function runs a generay query to Elasticsearch just to make sure we get a response
from the FETCH_INDEX.
Args:
es(Elasticsearch): an Elasticsearch object to which we run the test.
"""
try:
query = QueryString(query='*')
search = Search(using=es, index=FETCH_INDEX).query(query)[0:1]
response = search.execute().to_dict()
_, total_results = get_total_results(response)
except NotFoundError as e:
return_error("Fetch incidents test failed.\nError message: {}.".format(str(e).split(',')[2][2:-1]))