def run_query_command():
to_query = demisto.args()['query']
timestamp_from = demisto.args()['from']
timestamp_to = demisto.args().get('to', None)
write_context = demisto.args()['writeToContext'].lower()
time_range = get_time_range(timestamp_from, timestamp_to)
results = list(
ds.Reader(oauth_token=READER_OAUTH_TOKEN,
end_point=READER_ENDPOINT,
verify=not ALLOW_INSECURE).query(to_query,
start=float(time_range[0]),
stop=float(time_range[1]),
output='dict',
ts_format='iso'))
querylink = {
'DevoTableLink':
build_link(to_query, int(1000 * float(time_range[0])),
int(1000 * float(time_range[1])))
}
entry = {
'Type': entryTypes['note'],
'Contents': results,
'ContentsFormat': formats['json'],
'ReadableContentsFormat': formats['markdown']
}
entry_linq = {
'Type': entryTypes['note'],
'Contents': querylink,
'ContentsFormat': formats['json'],
'ReadableContentsFormat': formats['markdown']
}
if len(results) == 0:
entry['HumanReadable'] = 'No results found'
entry['Devo.QueryResults'] = None
entry['Devo.QueryLink'] = querylink
return entry
headers = list(results[0].keys())
md = tableToMarkdown('Devo query results', results, headers)
entry['HumanReadable'] = md
md_linq = tableToMarkdown(
'Link to Devo Query',
{'DevoTableLink': f'[Devo Direct Link]({querylink["DevoTableLink"]})'})
entry_linq['HumanReadable'] = md_linq
if write_context == 'true':
entry['EntryContext'] = {'Devo.QueryResults': createContext(results)}
entry_linq['EntryContext'] = {
'Devo.QueryLink': createContext(querylink)
}
return [entry, entry_linq]
def check_credentials():
results = list(ds.Reader(oauth_token=READER_OAUTH_TOKEN, end_point=READER_ENDPOINT)\
.query(HEALTHCHECK_QUERY, start=int(time.time() - 1), stop=int(time.time()), output='dict'))
if WRITER_RELAY and WRITER_CREDENTIALS:
creds = get_writer_creds()
linq = ds.Writer(key=creds['key'].name, crt=creds['crt'].name, chain=creds['chain'].name, relay=WRITER_RELAY)\
.load(HEALTHCHECK_WRITER_RECORD, HEALTHCHECK_WRITER_TABLE,
historical=False, linq_func=(lambda x: x))
return True
def parallel_query_helper(sub_query, append_list, timestamp_from,
timestamp_to):
append_list.extend(
list(
ds.Reader(oauth_token=READER_OAUTH_TOKEN,
end_point=READER_ENDPOINT).query(
sub_query,
start=float(timestamp_from),
stop=float(timestamp_to),
output='dict',
ts_format='iso')))
def multi_table_query_command():
tables_to_query = check_type(demisto.args()['tables'], list)
search_token = demisto.args()['searchToken']
timestamp_from = demisto.args()['from']
timestamp_to = demisto.args().get('to', None)
write_context = demisto.args()['writeToContext'].lower()
time_range = get_time_range(timestamp_from, timestamp_to)
futures = []
all_results: List[Dict] = []
sub_queries = []
for table in tables_to_query:
fields = ds.Reader(oauth_token=READER_OAUTH_TOKEN, end_point=READER_ENDPOINT)\
._get_types(f'from {table} select *', 'now', 'iso').keys()
clauses = [
f"( isnotnull({field}) and str({field})->\"" + search_token + "\")"
for field in fields
]
sub_queries.append("from " + table + " where" + " or ".join(clauses) +
" select *")
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
for q in sub_queries:
futures.append(
executor.submit(parallel_query_helper, q, all_results,
time_range[0], time_range[1]))
concurrent.futures.wait(futures)
entry = {
'Type': entryTypes['note'],
'Contents': all_results,
'ContentsFormat': formats['json'],
'ReadableContentsFormat': formats['markdown']
}
if len(all_results) == 0:
entry['HumanReadable'] = 'No results found'
return entry
headers: Set = set().union(*(r.keys() for r in all_results))
md = tableToMarkdown('Devo query results', all_results, headers)
entry['HumanReadable'] = md
if write_context == 'true':
entry['EntryContext'] = {
'Devo.MultiResults': createContext(all_results)
}
return entry
def check_configuration():
# Check all settings related if set
# Basic functionality of integration
list(
ds.Reader(oauth_token=READER_OAUTH_TOKEN,
end_point=READER_ENDPOINT,
verify=not ALLOW_INSECURE).query(HEALTHCHECK_QUERY,
start=int(time.time() - 1),
stop=int(time.time()),
output='dict'))
if WRITER_RELAY and WRITER_CREDENTIALS:
creds = get_writer_creds()
Sender(SenderConfigSSL(address=(WRITER_RELAY, PORT),
key=creds['key'].name, cert=creds['crt'].name, chain=creds['chain'].name))\
.send(tag=HEALTHCHECK_WRITER_TABLE, msg=f'{HEALTHCHECK_WRITER_RECORD}')
if FETCH_INCIDENTS_FILTER:
alert_filters = check_type(FETCH_INCIDENTS_FILTER, dict)
assert alert_filters['type'] in [
'AND', 'OR'
], 'Missing key:"type" or unsupported value in fetch_incidents_filters'
filters = check_type(alert_filters['filters'], list)
for filt in filters:
assert filt[
'key'], 'Missing key: "key" in fetch_incidents_filters.filters configuration'
assert filt['operator'] in ['=', '/=', '>', '<', '>=', '<=', 'and', 'or', '->'], 'Missing key: "operator"'\
' or unsupported operator in fetch_incidents_filters.filters configuration'
assert filt[
'value'], 'Missing key:"value" in fetch_incidents_filters.filters configuration'
if FETCH_INCIDENTS_DEDUPE:
dedupe_conf = check_type(FETCH_INCIDENTS_DEDUPE, dict)
assert isinstance(
dedupe_conf['cooldown'],
(int,
float)), 'Invalid fetch_incidents_deduplication configuration'
return True
def fetch_incidents():
last_run = demisto.getLastRun()
alert_query = ALERTS_QUERY
to_time = time.time()
if FETCH_INCIDENTS_FILTER:
alert_filters = check_type(FETCH_INCIDENTS_FILTER, list)
filter_string = ', '.join([
f'{filt["key"]} {filt["operator"]} "{urllib.parse.quote(filt["value"])}"'
for filt in alert_filters
])
alert_query = f'{alert_query} where {filter_string}'
from_time = to_time - 86400
if 'from_time' in last_run:
from_time = float(last_run['from_time'])
# execute the query and get the events
events = list(ds.Reader(oauth_token=READER_OAUTH_TOKEN, end_point=READER_ENDPOINT)\
.query(alert_query, start=float(from_time), stop=float(to_time),
output='dict', ts_format='timestamp'))
# convert the events to demisto incident
incidents = []
for event in events:
event['extraData'] = json.loads(event['extraData'])
for ed in event['extraData']:
event['extraData'][ed] = urllib.parse.unquote(
event['extraData'][ed])
inc = alert_to_incident(event)
incidents.append(inc)
demisto.setLastRun({'from_time': to_time})
# this command will create incidents in Demisto
demisto.incidents(incidents)
def get_alerts_command():
timestamp_from = demisto.args()['from']
timestamp_to = demisto.args().get('to', time.time())
alert_filters = demisto.args().get('filters', None)
write_context = demisto.args()['writeToContext'].lower()
alert_query = ALERTS_QUERY
if alert_filters:
alert_filters = check_type(alert_filters, list)
filter_string = ', '.join([
f'{filt["key"]} {filt["operator"]} "{urllib.parse.quote(filt["value"])}"'
for filt in alert_filters
])
alert_query = f'{alert_query} where {filter_string}'
results = list(ds.Reader(oauth_token=READER_OAUTH_TOKEN, end_point=READER_ENDPOINT)\
.query(alert_query, start=float(timestamp_from), stop=float(timestamp_to),
output='dict', ts_format='iso'))
querylink = {
'DevoTableLink':
build_link(alert_query, int(1000 * float(timestamp_from)),
int(1000 * float(timestamp_to)))
}
for res in results:
res['extraData'] = json.loads(res['extraData'])
for ed in res['extraData']:
res['extraData'][ed] = urllib.parse.unquote(res['extraData'][ed])
entry = {
'Type': entryTypes['note'],
'Contents': results,
'ContentsFormat': formats['json'],
'ReadableContentsFormat': formats['markdown']
}
entry_linq = {
'Type': entryTypes['note'],
'Contents': querylink,
'ContentsFormat': formats['json'],
'ReadableContentsFormat': formats['markdown']
}
if len(results) == 0:
entry['HumanReadable'] = 'No results found'
entry['Devo.AlertsResults'] = None
entry_linq['Devo.QueryLink'] = querylink
return entry
headers = list(results[0].keys())
md = tableToMarkdown('Devo query results', results, headers)
entry['HumanReadable'] = md
md_linq = tableToMarkdown(
'Link to Devo Query',
{'DevoTableLink': f'[Devo Direct Link]({querylink["DevoTableLink"]})'})
entry_linq['HumanReadable'] = md_linq
if write_context == 'true':
entry['EntryContext'] = {'Devo.AlertsResults': createContext(results)}
entry_linq['EntryContext'] = {
'Devo.QueryLink': createContext(querylink)
}
return [entry, entry_linq]
def fetch_incidents():
last_run = demisto.getLastRun()
alert_query = ALERTS_QUERY
to_time = time.time()
dedupe_config = None
alerts_list: Dict = {}
new_last_run: Dict = {'from_time': to_time}
if FETCH_INCIDENTS_FILTER:
alert_filters = check_type(FETCH_INCIDENTS_FILTER, dict)
if alert_filters['type'] == 'AND':
filter_string = ' , '.join([
f'{filt["key"]} {filt["operator"]} "{urllib.parse.quote(filt["value"])}"'
for filt in alert_filters['filters']
])
elif alert_filters['type'] == 'OR':
filter_string = ' or '.join([
f'{filt["key"]} {filt["operator"]} "{urllib.parse.quote(filt["value"])}"'
for filt in alert_filters['filters']
])
alert_query = f'{alert_query} where {filter_string}'
from_time = to_time - 3600
if 'from_time' in last_run:
from_time = float(last_run['from_time'])
if FETCH_INCIDENTS_DEDUPE:
dedupe_config = check_type(FETCH_INCIDENTS_DEDUPE, dict)
if 'alerts_list' in last_run:
alerts_list = last_run['alerts_list']
alerts_list = {
k: v
for k, v in alerts_list.items()
if alerts_list[k] >= (to_time - float(dedupe_config['cooldown']))
}
# execute the query and get the events
# reverse the list so that the most recent event timestamp event is taken when de-duping if needed.
events = list(
ds.Reader(oauth_token=READER_OAUTH_TOKEN,
end_point=READER_ENDPOINT).query(
alert_query,
start=float(from_time),
stop=float(to_time),
output='dict',
ts_format='timestamp'))[::-1]
deduped_events: List[Dict] = []
if FETCH_INCIDENTS_DEDUPE:
# Expire out of rolling time window events
for event in events:
if any(de['context'] == event['context'] for de in deduped_events):
continue
if event['context'] in alerts_list:
continue
deduped_events.append(event)
alerts_list[event['context']] = event['eventdate']
events = deduped_events
new_last_run['alerts_list'] = alerts_list
# convert the events to demisto incident
incidents = []
for event in events:
event['extraData'] = json.loads(event['extraData'])
for ed in event['extraData']:
event['extraData'][ed] = urllib.parse.unquote_plus(
event['extraData'][ed])
inc = alert_to_incident(event)
incidents.append(inc)
demisto.setLastRun(new_last_run)
# this command will create incidents in Demisto
demisto.incidents(incidents)
return incidents
