def fetch_incidents_command():
"""
Fetch alerts from Canary Tools as incidents in Demisto
last_fetch: The latest fetched alert creation time
"""
last_fetch = demisto.getLastRun().get('time')
if last_fetch is None:
last_fetch = parse_date_range(FETCH_DELTA, '%Y-%m-%d-%H:%M:%S')[0]
# All alerts retrieved from get_alerts are newer than last_fetch and are in a chronological order
alerts = get_alerts(last_fetch)
incidents = []
current_fetch = last_fetch
for alert in alerts:
current_fetch = 1000 * (
int(demisto.get(alert, 'description.created')) + 1)
current_fetch = timestamp_to_datestring(current_fetch,
'%Y-%m-%d-%H:%M:%S')
incident = create_incident(alert)
incidents.append(incident)
demisto.incidents(incidents)
demisto.setLastRun({'time': current_fetch})
def fetch_indicators_command(client, feed_type, src_val, src_type,
default_type, last_fetch):
"""Implements fetch-indicators command"""
last_fetch_timestamp = get_last_fetch_timestamp(last_fetch,
client.time_method,
client.fetch_time)
now = datetime.now()
ioc_lst: list = []
ioc_enrch_lst: list = []
if FEED_TYPE_GENERIC not in feed_type:
# Insight is the name of the indicator object as it's saved into the database
search = get_scan_insight_format(client, now, last_fetch_timestamp,
feed_type)
for hit in search.scan():
hit_lst, hit_enrch_lst = extract_indicators_from_insight_hit(hit)
ioc_lst.extend(hit_lst)
ioc_enrch_lst.extend(hit_enrch_lst)
else:
search = get_scan_generic_format(client, now, last_fetch_timestamp)
for hit in search.scan():
ioc_lst.extend(
extract_indicators_from_generic_hit(hit, src_val, src_type,
default_type))
if ioc_lst:
for b in batch(ioc_lst, batch_size=2000):
demisto.createIndicators(b)
if ioc_enrch_lst:
ioc_enrch_batches = create_enrichment_batches(ioc_enrch_lst)
for enrch_batch in ioc_enrch_batches:
# ensure batch sizes don't exceed 2000
for b in batch(enrch_batch, batch_size=2000):
demisto.createIndicators(b)
demisto.setLastRun({'time': now.timestamp() * 1000})
def main(): # pragma: no cover
params = demisto.params()
base_url = urljoin(params.get('url'), '/api/v2/')
verify_ssl = not params.get('insecure', False)
proxy = params.get('proxy')
client = Client(base_url=base_url, verify=verify_ssl, proxy=proxy)
command = demisto.command()
demisto.debug(f'Command being called is {command}')
# Switch case
commands = {
'test-module': test_module_command,
'fetch-incidents': fetch_incidents_command,
f'{INTEGRATION_COMMAND_NAME}-list-events': list_events_command,
f'{INTEGRATION_COMMAND_NAME}-get-event': get_event_command,
f'{INTEGRATION_COMMAND_NAME}-delete-event': close_event_command,
f'{INTEGRATION_COMMAND_NAME}-update-event': update_event_command,
f'{INTEGRATION_COMMAND_NAME}-create-event': create_event_command,
f'{INTEGRATION_COMMAND_NAME}-query': query_command
}
try:
if command == 'fetch-incidents':
incidents, new_last_run = fetch_incidents_command(
client, last_run=demisto.getLastRun())
demisto.incidents(incidents)
demisto.setLastRun(new_last_run)
elif command in commands:
readable_output, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
# Log exceptions
except Exception as e:
err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]'
return_error(err_msg, error=e)
def main():
"""
PARSE AND VALIDATE INTEGRATION PARAMS
"""
service_principal = demisto.params().get('credentials').get('identifier')
secret = demisto.params().get('credentials').get('password')
# Remove trailing slash to prevent wrong URL path to service
server_url = demisto.params()['url'][:-1] \
if (demisto.params()['url'] and demisto.params()['url'].endswith('/')) else demisto.params()['url']
api_version = demisto.params().get('api_version')
verify_certificate = not demisto.params().get('insecure', False)
# How many time before the first fetch to retrieve incidents
fetch_time = demisto.params().get('fetch_time', '3 days')
threat_status = argToList(demisto.params().get('threat_status'))
threat_type = argToList(demisto.params().get('threat_type'))
event_type_filter = demisto.params().get('events_type')
fetch_limit = 50
# Remove proxy if not set to true in params
proxies = handle_proxy()
LOG('Command being called is %s' % (demisto.command()))
try:
client = Client(server_url, api_version, verify_certificate,
service_principal, secret, proxies)
if demisto.command() == 'test-module':
results = test_module(client, fetch_time, event_type_filter)
return_outputs(results, None)
elif demisto.command() == 'fetch-incidents':
integration_context = demisto.getIntegrationContext()
next_run, incidents, remained_incidents = fetch_incidents(
client=client,
last_run=demisto.getLastRun(),
first_fetch_time=fetch_time,
event_type_filter=event_type_filter,
threat_status=threat_status,
threat_type=threat_type,
limit=fetch_limit,
integration_context=integration_context)
# Save last_run, incidents, remained incidents into integration
demisto.setLastRun(next_run)
demisto.incidents(incidents)
# preserve context dict
integration_context['incidents'] = remained_incidents
demisto.setIntegrationContext(integration_context)
elif demisto.command() == 'proofpoint-get-events':
return_outputs(*get_events_command(client, demisto.args()))
except Exception as e:
return_error(str(e))
def main():
params = demisto.params()
client = Client(base_url=urljoin(params.get('host'), '/siem/v1/configs'),
verify=not params.get('insecure', False),
proxy=params.get('proxy'),
auth=EdgeGridAuth(
client_token=params.get('clientToken'),
access_token=params.get('accessToken'),
client_secret=params.get('clientSecret')))
commands = {
f"test-module": test_module_command,
f"{INTEGRATION_COMMAND_NAME}-get-events": get_events_command
}
command = demisto.command()
demisto.debug(f'Command being called is {command}')
try:
if command == 'fetch-incidents':
incidents, new_last_run = fetch_incidents_command(
client,
fetch_time=params.get('fetchTime'),
fetch_limit=params.get('fetchLimit'),
config_ids=params.get('configIds'),
last_run=demisto.getLastRun().get('lastRun'))
demisto.incidents(incidents)
demisto.setLastRun(new_last_run)
else:
human_readable, entry_context, raw_response = commands[command](
client, **demisto.args())
return_outputs(human_readable, entry_context, raw_response)
except Exception as e:
err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]'
return_error(err_msg, error=e)
def update_indicators_iterator(indicators_iterator: List[Dict[str, Any]],
params_dict: Dict[str, Any],
is_fetch: bool) -> Optional[List[Dict[str, Any]]]:
"""
sorts the indicators by their timestamp and returns a list of only new indicators received from MISP
Args:
params_dict: user's params sent to misp
indicators_iterator: list of indicators
is_fetch: flag for wether funciton was called for fetching command or a get
Returns: Sorted list of new indicators
"""
last_run = demisto.getLastRun()
indicators_iterator.sort(key=lambda indicator: indicator['value']['timestamp'])
if last_run is None:
return indicators_iterator
if params_dict != last_run.get('params'):
if is_fetch:
demisto.setLastRun(None)
return indicators_iterator
last_timestamp = int(last_run.get('timestamp'))
for index in range(len(indicators_iterator)):
if int(indicators_iterator[index]['value']['timestamp']) > last_timestamp:
return indicators_iterator[index:]
return []
def fetch_incidents():
param_dict = {}
now_time = datetime.utcnow()
now = datetime.isoformat(now_time)
lastRunObject = demisto.getLastRun()
if lastRunObject:
param_dict['since'] = lastRunObject['time']
else:
param_dict['since'] = datetime.isoformat(now_time - timedelta(minutes=int(FETCH_INTERVAL)))
param_dict['until'] = now
url = SERVER_URL + GET_INCIDENTS_SUFFIX + configure_status()
res = http_request('GET', url, param_dict)
_, parsed_incidents, raw_responses = parse_incident_data(res.get('incidents', []))
incidents = []
for incident, raw_response in zip(parsed_incidents, raw_responses):
incidents.append({
'name': incident['ID'] + ' - ' + incident['Title'],
'occurred': incident['created_at'],
'severity': translate_severity(incident['urgency']),
'rawJSON': json.dumps(raw_response)
})
demisto.incidents(incidents)
demisto.setLastRun({'time': now})
def fetch_incidents(client: Client, first_fetch_time, fetch_limit, fetch_state,
fetch_severity, fetch_status, fetch_app_ids):
last_run = demisto.getLastRun()
last_fetch = last_run.get('last_run_time')
if last_fetch is None:
last_fetch = dateparser.parse(first_fetch_time,
settings={'TIMEZONE': 'UTC'})
last_fetch = last_fetch.strftime(
SAAS_SECURITY_DATE_FORMAT
)[:-4] + 'Z' # format ex: 2021-08-23T09:26:25.872Z
current_fetch = last_fetch
results = client.get_incidents(limit=fetch_limit,
from_time=last_fetch,
state=fetch_state,
severity=fetch_severity,
status=fetch_status,
app_ids=fetch_app_ids).get('resources', [])
incidents = list()
for inc in results:
incident = convert_to_xsoar_incident(inc)
incidents.append(incident)
date_updated = inc.get('updated_at')
date_updated_dt = datetime.strptime(date_updated,
SAAS_SECURITY_DATE_FORMAT)
if date_updated_dt > datetime.strptime(current_fetch,
SAAS_SECURITY_DATE_FORMAT):
current_fetch = date_updated
demisto.setLastRun({'last_run_time': current_fetch})
demisto.incidents(incidents)
def fetch_incidents():
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch = datetime.now() - timedelta(days=5)
else:
last_fetch = datetime.strptime(last_fetch, '%Y-%m-%dT%H:%M:%SZ')
LOG('iterating on detections, looking for more recent than {}'.format(
last_fetch))
incidents = []
for raw_detection in get_unacknowledge_detections(last_fetch, per_page=2):
LOG('found detection #{}'.format(raw_detection['id']))
incident = detection_to_incident(raw_detection)
incidents.append(incident)
if len(incidents) != 0:
last_fetch = max([
get_time_obj(incident['occurred']) for incident in incidents
]) # noqa:F812
demisto.setLastRun(
{'time': get_time_str(last_fetch + timedelta(seconds=1))})
demisto.incidents(incidents)
def get_no_update_value(response: requests.models.Response, url: str) -> bool:
"""
detect if the feed response has been modified according to the headers etag and last_modified.
For more information, see this:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
Args:
response: (requests.Response) The feed response.
url: (str) The feed URL (service).
Returns:
boolean with the value for noUpdate argument.
The value should be False if the response was modified.
"""
if response.status_code == 304:
demisto.debug('No new indicators fetched, createIndicators will be executed with noUpdate=True.')
return True
etag = response.headers.get('ETag')
last_modified = response.headers.get('Last-Modified')
if not etag and not last_modified:
demisto.debug('Last-Modified and Etag headers are not exists,'
'createIndicators will be executed with noUpdate=False.')
return False
last_run = demisto.getLastRun()
last_run[url] = {'last_modified': last_modified, 'etag': etag}
demisto.setLastRun(last_run)
demisto.debug('New indicators fetched - the Last-Modified value has been updated,'
' createIndicators will be executed with noUpdate=False.')
return False
def main(command: str, demisto_params: dict):
demisto.debug(f'Command being called is {command}')
try:
demisto_params['client_secret'] = demisto_params['credentials'][
'password']
push_to_xsiam = argToBoolean(
demisto_params.get('should_push_events', 'false'))
vendor = demisto_params.get('vendor')
product = demisto_params.get('product')
after = demisto_params.get('after')
if after and not isinstance(after, int):
timestamp = dateparser.parse(after)
after = int(timestamp.timestamp() * 1000) # type: ignore
options = IntegrationOptions.parse_obj(demisto_params)
request = DefenderHTTPRequest.parse_obj(demisto_params)
authenticator = DefenderAuthenticator.parse_obj(demisto_params)
client = DefenderClient(request=request,
options=options,
authenticator=authenticator,
after=after)
get_events = DefenderGetEvents(client=client, options=options)
if command == 'test-module':
return_results(test_module(get_events=get_events))
elif command in ('fetch-events',
'microsoft-defender-cloud-apps-get-events'):
events = get_events.run()
if command == 'fetch-events':
# publishing events to XSIAM
send_events_to_xsiam(events, vendor=vendor, product=product)
demisto.setLastRun(DefenderGetEvents.get_last_run(events))
elif command == 'microsoft-defender-cloud-apps-get-events':
command_results = CommandResults(
readable_output=tableToMarkdown(
'microsoft defender cloud apps events',
events,
headerTransform=pascalToSpace),
outputs_prefix='Microsoft.Events',
outputs_key_field='_id',
outputs=events,
raw_response=events,
)
return_results(command_results)
if push_to_xsiam:
# publishing events to XSIAM
send_events_to_xsiam(events,
vendor=vendor,
product=product)
# Log exceptions and return errors
except Exception as e:
demisto.error(traceback.format_exc()) # print the traceback
return_error(f'Failed to execute {command} command.\nError:\n{str(e)}')
def fetch_incidents():
last_run = demisto.getLastRun()
if last_run and last_run['last_alert_fetched_time']:
last_alert_fetched_time = datetime.strptime(last_run['last_alert_fetched_time'], '%Y-%m-%dT%H:%M:%S.%f')
else:
last_alert_fetched_time = datetime.now() - timedelta(days=300)
latest_creation_time = last_alert_fetched_time
alerts = list_alerts()['value']
incidents = []
for alert in alerts:
# Removing 'Z' from timestamp and converting to datetime
alert_creation_time = datetime.strptime(alert['alertCreationTime'][:-2], '%Y-%m-%dT%H:%M:%S.%f')
alert_status = alert['status']
alert_severity = alert['severity']
if alert_creation_time > last_alert_fetched_time and alert_status in FETCH_STATUS and \
alert_severity in FETCH_SEVERITY:
incident = alert_to_incident(alert)
incidents.append(incident)
if alert_creation_time > latest_creation_time:
latest_creation_time = alert_creation_time
demisto.setLastRun({
'last_alert_fetched_time': datetime.strftime(latest_creation_time, '%Y-%m-%dT%H:%M:%S.%f')
})
demisto.incidents(incidents)
def fetch_incidents():
last_run = demisto.getLastRun()
# Get the last fetch time, if exists
last_fetch = last_run.get('time')
# Handle first time fetch, fetch incidents retroactively
if last_fetch is None:
last_fetch, _ = parse_date_range(FETCH_TIME, date_format='%Y-%m-%dT%H:%M:%S')
latest = datetime.strptime(last_fetch, '%Y-%m-%dT%H:%M:%S')
demisto.debug('getting alarms since {}'.format(last_fetch))
incidents = []
items = list_alerts(time_frame='Custom', start_time=last_fetch)
demisto.debug('got {} new alarms'.format(len(items)))
for item in items:
incident_date = datetime.strptime(item['ALERT_TIME'], '%Y-%m-%d %H:%M:%S')
incident = {
'Type': 'Fidelis',
'name': '{} {}'.format(item['ALERT_ID'], item['SUMMARY']),
'occurred': incident_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
'rawJSON': json.dumps(item),
}
latest = max(latest, incident_date)
incidents.append(incident)
if latest != last_fetch:
last_fetch = (latest + timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S')
demisto.setLastRun({'time': last_fetch})
demisto.incidents(incidents)
def fetch_incidents_command(client, params, is_test_module=False):
list_services = params.get('service')
list_event_type = params.get('event_type')
first_fetch_time = parse_date_to_isoformat(
params.get('first_fetch', '3 days'), 'first_fetch_time')
max_results = int(params.get('max_fetch', 50))
if not max_results or max_results > MAX_INCIDENTS_TO_FETCH:
max_results = MAX_INCIDENTS_TO_FETCH
last_run = demisto.getLastRun() # getLastRun() gets the last run dict
if not last_run:
last_run = creates_empty_dictionary_of_last_run(
list_services, list_event_type)
next_run, incidents = fetch_incidents(
client=client,
max_results=max_results,
last_run=last_run, # getLastRun() gets the last run dict
list_services=list_services,
list_event_type=list_event_type,
first_fetch_time=first_fetch_time,
is_test_module=is_test_module)
if is_test_module:
return
demisto.setLastRun(next_run)
demisto.incidents(incidents)
def fetch_incidents(self, params: Dict):
last_run = demisto.getLastRun()
current_run = {}
incidents = []
if params.get('fetchType', 'alarms') in ('alarms', 'both'):
start_time = last_run.get('alarms', {}).get(
'time',
parse_date_range(params.get('fetchTime'),
self.demisto_format)[0])
start_id = int(
last_run.get('alarms', {}).get('id',
params.get('startingFetchID')))
temp_incidents, current_run['alarms'] = \
self.__alarms_to_incidents(start_time, start_id, int(params.get('fetchLimitAlarms', 5)))
incidents.extend(temp_incidents)
if params.get('fetchType') in ('cases', 'both'):
start_id = int(
last_run.get('cases', {}).get('id',
params.get('startingFetchID')))
temp_incidents, current_run['cases'] = \
self.__cases_to_incidents(start_id=start_id, limit=int(params.get('fetchLimitCases', 5)))
incidents.extend(temp_incidents)
demisto.setLastRun(current_run)
demisto.incidents(incidents)
def fetch_incidents(client, alert_type):
"""
fetches alerts and turns them into incidents.
"""
last_run = demisto.getLastRun()
# Get the last fetch time, if exists
last_fetch = last_run.get('time')
# Handle first time fetch, fetch incidents retroactively
if last_fetch is None:
fetch_time, _ = parse_date_range(FETCH_TIME)
last_fetch = fetch_time.strftime(DATE_FORMAT)
incidents = []
###
items = iter_all_alerts(client,
alert_type,
last_fetch,
datetime.now().strftime(DATE_FORMAT),
severity=SEVERITY_TYPE)
###
for item in items:
incident = item_to_incident(item)
incident_date = datetime.strptime(incident['timestamp'], DATE_FORMAT)
# Update last run and add incident if the incident is newer than last fetch
if incident_date.timestamp() > datetime.strptime(
last_fetch, DATE_FORMAT).timestamp():
last_fetch = incident_date.strftime(DATE_FORMAT)
incidents.append(incident)
demisto.setLastRun({'time': last_fetch})
demisto.incidents(incidents)
def fetch_incidents(client: Client,
fetch_time: str,
fetch_shaping: str,
last_run: Dict,
fetch_limit: str,
fetch_queue_id: Optional[list] = None,
fetch_filter: Optional[str] = None) -> list:
"""
This function will execute each interval (default is 1 minute).
Args:
client (Client): Quest Kace Client
fetch_time: time interval for fetch incidents.
fetch_shaping: shaping for the request.
fetch_filter: custom filters for the request.
fetch_limit: limit for number of fetch incidents per fetch.
fetch_queue_id: queue id for fetch, if not given then fetch runs on all tickets in the system
last_run (dateparser.time): The greatest incident created_time we fetched from last fetch
Returns:
incidents: Incidents that will be created in Demisto
"""
if not fetch_queue_id or fetch_queue_id[0] == 'All':
fetch_queue_id = get_queue_ids(client)
time_format = '%Y-%m-%dT%H:%M:%SZ'
if not last_run: # if first time running
new_last_run = {
'last_fetch': parse_date_range(fetch_time,
date_format=time_format)[0]
}
else:
new_last_run = last_run
if not fetch_shaping:
fetch_shaping = shaping_fetch(client, fetch_queue_id)
parsed_last_time = datetime.strptime(new_last_run.get('last_fetch', ''),
time_format)
fetch_filter_for_query = f'created gt {parsed_last_time}'
if fetch_queue_id:
queue_id_str = ';'.join(fetch_queue_id)
filter_by_queue_id = f'hd_queue_id in {queue_id_str}'
fetch_filter_for_query = f'{fetch_filter_for_query},{filter_by_queue_id}'
if fetch_filter:
fetch_filter_for_query = f'{fetch_filter_for_query},{fetch_filter}'
demisto.info(f"Fetching Incident has Started,\n"
f"Fetch filter is {fetch_filter_for_query}\n"
f"Last fetch was on {str(parsed_last_time)}")
client.update_token()
items: dict = client.tickets_list_request(fetch_shaping,
fetch_filter_for_query)
items: list = items.get('Tickets', [])
incidents, last_incident_time = parse_incidents(items, fetch_limit,
time_format,
parsed_last_time)
last_incident_time = last_incident_time.strftime(time_format)
demisto.info(f"Fetching Incident has Finished\n"
f"Fetch limit was {fetch_limit}"
f"Last fetch was on {str(last_incident_time)}\n"
f"Number of incidents was {len(incidents)}")
demisto.setLastRun({'last_fetch': last_incident_time})
return incidents
def main():
# Write configure here
params = {k: v for k, v in demisto.params().items() if v is not None}
handle_proxy()
client = TAXIIClient(**params)
command = demisto.command()
demisto.info('Command being called is {command}'.format(command=command))
# Switch case
commands = {
'test-module': test_module,
'get-indicators': get_indicators_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
demisto.setLastRun({'time': client.last_taxii_run})
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]'
raise Exception(err_msg)
def fetch_incidents(client, params):
now = datetime.utcnow()
incidents = []
fetch_params = _set_fetch_params(params, now)
fetch_passwords = params.get('fetch_passwords')
severity = getattr(IncidentSeverity, params['fetch_severity'].upper())
incident_type = params.get('incidentType')
if not incident_type or incident_type == '':
incident_type = INCIDENT_TYPE
response = client.search('exposures', fetch_params)
if response and response.get('data'):
data = _process_exposure_data(response['data'], fetch_passwords)
i = {
'name': INCIDENT_NAME,
'occurred': now.strftime(ISO8601_FORMAT),
'rawJSON': json.dumps(data),
'type': incident_type,
'severity': severity
}
incidents.append(i)
demisto.setLastRun({'start_time': now.timestamp()})
return incidents
def fetch_incidents_command():
last_run = demisto.getLastRun()
if last_run and 'start_time' in last_run:
start_time = datetime.strptime(last_run.get('start_time'), '%Y-%m-%dT%H:%M:%SZ')
else:
start_time = datetime.now() - timedelta(days=int(FETCH_TIME))
last_time = start_time
issue_list = http_request(method='GET',
url_suffix=ISSUE_SUFFIX,
params={'state': 'all'})
incidents = []
for issue in issue_list:
updated_at_str = issue.get('created_at')
updated_at = datetime.strptime(updated_at_str, '%Y-%m-%dT%H:%M:%SZ')
if updated_at > start_time:
inc = {
'name': issue.get('url'),
'occurred': updated_at_str,
'rawJSON': json.dumps(issue)
}
incidents.append(inc)
if updated_at > last_time:
last_time = updated_at
demisto.setLastRun({'start_time': datetime.strftime(last_time, '%Y-%m-%dT%H:%M:%SZ')})
demisto.incidents(incidents)
def fetch_incidents(query, id_offset, fetch_by_created=None, **_):
last_run = demisto.getLastRun()
demisto.debug(f"last_run: {last_run}" if last_run else 'last_run is empty')
if last_run and last_run.get("idOffset"):
id_offset = last_run.get("idOffset")
if not id_offset:
id_offset = 0
incidents, max_results = [], 50
if id_offset:
query = f'{query} AND id >= {id_offset}'
if fetch_by_created:
query = f'{query} AND created>-1m'
res = run_query(query, '', max_results)
curr_id = id_offset
for ticket in res.get('issues'):
ticket_id = int(ticket.get("id"))
if ticket_id == curr_id:
continue
id_offset = max(int(id_offset), ticket_id)
incidents.append(create_incident_from_ticket(ticket))
demisto.setLastRun({"idOffset": id_offset})
demisto.incidents(incidents)
def fetch_incidents():
last_run = demisto.getLastRun()
# Get the last fetch time, if exists
last_fetch = last_run.get('time')
# Handle first time fetch, fetch incidents retroactively
if last_fetch is None:
last_fetch, _ = parse_date_range(FETCH_TIME,
date_format='%Y-%m-%dT%H:%M:%S')
latest = datetime.strptime(last_fetch, '%Y-%m-%dT%H:%M:%S')
demisto.debug('getting alarms since {}'.format(last_fetch))
incidents = []
items = list_alerts(time_frame='Custom', start_time=last_fetch)
demisto.debug('got {} new alarms'.format(len(items)))
for item in items:
incident_date = datetime.strptime(item['ALERT_TIME'],
'%Y-%m-%d %H:%M:%S')
incident = {
'Type': 'Fidelis',
'name': '{} {}'.format(item['ALERT_ID'], item['SUMMARY']),
'occurred': incident_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
'rawJSON': json.dumps(item),
}
latest = max(latest, incident_date)
incidents.append(incident)
if latest != last_fetch:
last_fetch = (latest +
timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S')
demisto.setLastRun({'time': last_fetch})
demisto.incidents(incidents)
def main():
params = demisto.params()
username = params.get('credentials').get('identifier')
password = params.get('credentials').get('password')
server = urljoin(params.get('url'), '/api/v2.0/')
# Should we use SSL
use_ssl = not params.get('insecure') == 'true'
use_proxy = params.get('proxy') == 'true'
# Headers to be sent in requests
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
client = Client(server,
use_ssl,
use_proxy,
headers=headers,
auth=(username, password))
# Commands switch case
command = demisto.command()
commands = {f'{INTEGRATION_COMMAND_NAME}-query': query_command}
if command == 'fetch-incidents':
# How many time before the first fetch to retrieve incidents
fetch_time = params.get('fetch_time', '3 days')
columns = params.get('columns')
table_name = params.get('table_name')
date_name = params.get('date_name')
last_run, incidents = fetch_incidents_command(client,
demisto.getLastRun(),
fetch_time, table_name,
columns, date_name)
demisto.setLastRun(last_run)
demisto.incidents(incidents)
elif command in commands:
return_outputs(*commands[command](client, demisto.args()))
def fetch_incidents(service):
last_run = demisto.getLastRun() and demisto.getLastRun()['time']
search_offset = demisto.getLastRun().get('offset', 0)
incidents = []
current_time_for_fetch = datetime.utcnow()
dem_params = demisto.params()
if demisto.get(dem_params, 'timezone'):
timezone = dem_params['timezone']
current_time_for_fetch = current_time_for_fetch + timedelta(
minutes=int(timezone))
now = current_time_for_fetch.strftime(SPLUNK_TIME_FORMAT)
if demisto.get(dem_params, 'useSplunkTime'):
now = get_current_splunk_time(service)
current_time_in_splunk = datetime.strptime(now, SPLUNK_TIME_FORMAT)
current_time_for_fetch = current_time_in_splunk
if len(last_run) == 0:
fetch_time_in_minutes = parse_time_to_minutes()
start_time_for_fetch = current_time_for_fetch - timedelta(
minutes=fetch_time_in_minutes)
last_run = start_time_for_fetch.strftime(SPLUNK_TIME_FORMAT)
earliest_fetch_time_fieldname = dem_params.get(
"earliest_fetch_time_fieldname", "index_earliest")
latest_fetch_time_fieldname = dem_params.get("latest_fetch_time_fieldname",
"index_latest")
kwargs_oneshot = {
earliest_fetch_time_fieldname: last_run,
latest_fetch_time_fieldname: now,
"count": FETCH_LIMIT,
'offset': search_offset
}
searchquery_oneshot = dem_params['fetchQuery']
if demisto.get(dem_params, 'extractFields'):
extractFields = dem_params['extractFields']
extra_raw_arr = extractFields.split(',')
for field in extra_raw_arr:
field_trimmed = field.strip()
searchquery_oneshot = searchquery_oneshot + ' | eval ' + field_trimmed + '=' + field_trimmed
oneshotsearch_results = service.jobs.oneshot(
searchquery_oneshot, **kwargs_oneshot) # type: ignore
reader = results.ResultsReader(oneshotsearch_results)
for item in reader:
inc = notable_to_incident(item)
incidents.append(inc)
demisto.incidents(incidents)
if len(incidents) < FETCH_LIMIT:
demisto.setLastRun({'time': now, 'offset': 0})
else:
demisto.setLastRun({
'time': last_run,
'offset': search_offset + FETCH_LIMIT
})
def fetch_incidents():
query = demisto.params().get('query')
last_run = demisto.getLastRun()
offense_id = last_run['id'] if last_run and 'id' in last_run else 0
if last_run and offense_id == 0:
start_time = last_run['startTime'] if 'startTime' in last_run else '0'
fetch_query = 'start_time>{0}{1}'.format(
start_time, ' AND ({0})'.format(query) if query else '')
else:
fetch_query = 'id>{0} {1}'.format(
offense_id, 'AND ({0})'.format(query) if query else '')
# qradar returns offenses sorted desc on id and there's no way to change sorting.
# if we get `offensesPerCall` offenses it means we (probably) have more than that so we
# start looking for the end of the list by doubling the page position until we're empty.
# then start binary search back until you find the end of the list and finally return
# `offensesPerCall` from the end.
raw_offenses = get_offenses(_range='0-{0}'.format(OFFENSES_PER_CALL),
_filter=fetch_query)
if len(raw_offenses) >= OFFENSES_PER_CALL:
last_offense_pos = find_last_page_pos(fetch_query)
raw_offenses = get_offenses(_range='{0}-{1}'.format(
last_offense_pos - OFFENSES_PER_CALL + 1, last_offense_pos),
_filter=fetch_query)
raw_offenses = unicode_to_str_recur(raw_offenses)
incidents = []
enrich_offense_res_with_source_and_destination_address(raw_offenses)
for offense in raw_offenses:
offense_id = max(offense_id, offense['id'])
incidents.append(create_incident_from_offense(offense))
demisto.setLastRun({'id': offense_id})
return incidents
def fetch_incidents():
"""
Fetches incidents
"""
incidents = list()
# Check for topic in Kafka
if TOPIC in KAFKA_CLIENT.topics:
kafka_topic = KAFKA_CLIENT.topics[TOPIC]
offset, partition = check_params(kafka_topic,
old_offset=OFFSET,
old_partition=PARTITION)
last_offset = demisto.getLastRun().get('last_offset')
last_fetch = last_offset if last_offset > offset else offset
# If need to fetch
latest_offset = check_latest_offset(kafka_topic, partition)
if latest_offset > last_fetch:
consumer = kafka_topic.get_simple_consumer(
auto_offset_reset=last_fetch, reset_offset_on_start=True)
for i in range(last_fetch, latest_offset):
"""
consumer.consume() will consume only one message from the given offset.
"""
message = consumer.consume()
incidents.append(create_incident(message=message, topic=TOPIC))
if message.offset == latest_offset:
break
demisto.setLastRun({'last_offset': latest_offset})
demisto.incidents(incidents)
else:
return_error(
'No such topic \'{}\' to fetch incidents from.'.format(TOPIC))
def fetch_incidents():
last_run = demisto.getLastRun()
if last_run and 'time' in last_run:
last_fetch = last_run.get('time')
last_fetch = datetime.strptime(last_fetch, TIME_FORMAT)
else:
last_fetch = parse_date_range(
demisto.params().get('fetch_time', '3 days'), TIME_FORMAT)[0]
LOG('iterating on detections, looking for more recent than {}'.format(
last_fetch))
incidents = []
for raw_detection in get_unacknowledged_detections(last_fetch, per_page=2):
LOG('found detection #{}'.format(raw_detection['id']))
incident = detection_to_incident(raw_detection)
incidents.append(incident)
if incidents:
last_fetch = max([
get_time_obj(incident['occurred']) for incident in incidents
]) # noqa:F812
demisto.setLastRun(
{'time': get_time_str(last_fetch + timedelta(seconds=1))})
demisto.incidents(incidents)
def fetch_incidents(
client: Client,
first_fetch: str,
dashboard_id: str = None,
panel_id: str = None,
alert_name: str = None,
state: str = None,
limit: Optional[int] = MAX_INCIDENTS_TO_FETCH) -> List[dict]:
limit = limit or MAX_INCIDENTS_TO_FETCH
last_fetch = demisto.getLastRun().get('last_fetch')
last_id_fetched = demisto.getLastRun().get('last_id_fetched', -1)
fetch_start_time = calculate_fetch_start_time(last_fetch, first_fetch)
demisto.debug(
f'last fetch was at: {last_fetch}, last id fetched was: {last_id_fetched}, '
f'time to fetch from is: {fetch_start_time}')
alerts = client.alerts_list_request(dashboard_id=argToList(dashboard_id),
panel_id=panel_id,
query=alert_name,
state=argToList(state))
last_fetch, last_id_fetched, incidents = parse_alerts(
alerts, limit, fetch_start_time, last_id_fetched)
demisto.debug(
f'last fetch now is: {last_fetch}, last id fetched is now: {last_id_fetched}, '
f'number of incidents fetched is {len(incidents)}, number of alerts got is {len(alerts)}'
)
demisto.setLastRun({
'last_fetch':
str(date_to_timestamp(last_fetch, DATE_FORMAT)),
'last_id_fetched':
last_id_fetched
})
return incidents
def fetch_incidents():
param_dict = {}
now_time = datetime.utcnow()
now = datetime.isoformat(now_time)
lastRunObject = demisto.getLastRun()
if lastRunObject:
param_dict['since'] = lastRunObject['time']
else:
param_dict['since'] = datetime.isoformat(now_time - timedelta(
minutes=int(FETCH_INTERVAL)))
param_dict['until'] = now
url = SERVER_URL + GET_INCIDENTS_SUFFIX + configure_status()
res = http_request('GET', url, param_dict)
_, parsed_incidents, raw_responses = parse_incident_data(
res.get('incidents', []))
incidents = []
for incident, raw_response in zip(parsed_incidents, raw_responses):
incidents.append({
'name': incident['ID'] + ' - ' + incident['Title'],
'occurred': incident['created_at'],
'severity': translate_severity(incident['urgency']),
'rawJSON': json.dumps(raw_response)
})
demisto.incidents(incidents)
demisto.setLastRun({'time': now})
def fetch_incidents(client: Client):
last_run = demisto.getLastRun()
last_timestamp = int(last_run.get('timestamp', 0))
res = client.get_cases()
res[:] = [
x for x in res
if x['createdAt'] > last_timestamp and x['status'] == 'Open'
]
res = sorted(res, key=lambda x: x['createdAt'])
incidents = list()
instance_name = demisto.integrationInstance()
mirror_direction = demisto.params().get('mirror')
mirror_direction = None if mirror_direction == "Disabled" else mirror_direction
for case in res:
case['dbotMirrorDirection'] = mirror_direction
case['dbotMirrorInstance'] = instance_name
incident = {
'name': case['title'],
'occurred': timestamp_to_datestring(case['createdAt']),
'severity': case['severity'],
'rawJSON': json.dumps(case)
}
incidents.append(incident)
last_timestamp = case['createdAt'] if case[
'createdAt'] > last_timestamp else last_timestamp
demisto.setLastRun({"timestamp": str(last_timestamp)})
return incidents
def fetch_incidents():
date_format = '%Y-%m-%dT%H:%M:%S'
last_run = demisto.getLastRun()
if last_run and last_run.get('last_fetched_event_timestamp'):
last_update_time = last_run['last_fetched_event_timestamp']
else:
last_update_time = parse_date_range(FETCH_TIME, date_format=date_format)[0]
incidents = []
limit: int = int(demisto.params().get('fetch_limit', ''))
response_content = list_alerts({'sort_direction': 'asc', 'limit': limit, 'last_modified_min_date': last_update_time})
alerts: List = response_content.get('alerts', [])
if alerts:
for alert in alerts:
incident = alert_to_incident(alert)
incidents.append(incident)
# max_update_time is the timestamp of the last alert in alerts (alerts is a sorted list)
last_alert_timestamp: str = str(alerts[-1].get('timestamp', ''))
# ZeroFox is using full timestamp ISO 8061 format which includes the GMT field i.e (+00:00/-00:00)
# The option to refine the search of alerts in the API is by the ISO 8061 format without the GMT field.
if '+' in last_alert_timestamp:
max_update_time = last_alert_timestamp.split('+')[0]
else:
max_update_time = last_alert_timestamp.split('-')[0]
# add 1 second to last alert timestamp, in order to prevent duplicated alerts
max_update_time = (datetime.strptime(max_update_time, date_format) + timedelta(seconds=1)).isoformat()
demisto.setLastRun({'last_fetched_event_timestamp': max_update_time})
demisto.incidents(incidents)
def fetch_incidents(client, first_fetch, max_fetch):
"""
This function retrieves new incidents every interval (default is 1 minute).
"""
now = datetime.timestamp(datetime.utcnow())
last_run_object = demisto.getLastRun()
last_run = last_run_object.get('time', None) if last_run_object else None
if not last_run:
if first_fetch:
last_run = float(first_fetch)
else:
last_run = datetime.timestamp(datetime.utcnow() - timedelta(days=1))
result = client.get_incidents(last_run, now)
if not result.get('success'):
raise DemistoException(f"ERROR: {result.get('message')}; last_run: {last_run}; now: {now}")
lp_incidents = result.get('incidents')
incidents = []
if len(lp_incidents) > max_fetch:
next_fetch_time = lp_incidents[max_fetch]['detection_timestamp']
lp_incidents = lp_incidents[:max_fetch]
else:
next_fetch_time = now
demisto.info(f"Executing LogPoint fetch_incidents between {last_run} and {next_fetch_time} Timestamp.")
for inc in lp_incidents:
detection_ts = inc['detection_timestamp']
dt = datetime.utcfromtimestamp(detection_ts)
occurred = dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
incidents.append({
'name': inc.get('name', 'LogPoint - No name'),
'occurred': occurred,
'severity': get_demisto_severity(inc.get('risk_level')),
'rawJSON': json.dumps(inc)
})
demisto.setLastRun({'time': next_fetch_time})
return incidents
def fetch():
"""
Query viewer should be defined in ArcSight ESM. fetch incidents fetches the results of query viewer
and converts them to Demisto incidents. We can query Cases or Events. If Cases are fetched then the
query viewer query must return fields ID and Create Time. If Events are fetched then Event ID and Start Time.
"""
events_query_viewer_id = demisto.params().get('viewerId')
cases_query_viewer_id = demisto.params().get('casesQueryViewerId')
type_of_incident = 'case' if events_query_viewer_id else 'event'
last_run = json.loads(demisto.getLastRun().get('value', '{}'))
already_fetched = last_run.get('already_fetched', [])
fields, query_results = get_query_viewer_results(events_query_viewer_id or cases_query_viewer_id)
# sort query_results by creation time
query_results.sort(key=lambda k: int(k.get('Start Time') or k.get('Create Time')))
incidents = []
for result in query_results:
# convert case or event to demisto incident
r_id = result.get('ID') or result.get('Event ID')
if r_id not in already_fetched:
create_time_epoch = int(result.get('Start Time') or result.get('Create Time'))
result['Create Time'] = parse_timestamp_to_datestring(create_time_epoch)
incident_name = result.get('Name') or 'New {} from arcsight at {}'.format(type_of_incident, datetime.now())
labels = [{'type': key.encode('utf-8'), 'value': value.encode('utf-8') if value else value} for key, value
in result.items()]
incident = {
'name': incident_name,
'occurred': result['Create Time'],
'labels': labels,
'rawJSON': json.dumps(result)
}
incidents.append(incident)
if len(incidents) >= FETCH_CHUNK_SIZE:
break
if len(already_fetched) > MAX_UNIQUE:
already_fetched.pop(0)
already_fetched.append(r_id)
last_run = {
'already_fetched': already_fetched,
}
demisto.setLastRun({'value': json.dumps(last_run)})
decode_arcsight_output(incidents)
if demisto.command() == 'as-fetch-incidents':
contents = {
'last_run': last_run,
'last_run_updated': demisto.getLastRun(),
'incidents': incidents,
'already_fetched': already_fetched
}
return_outputs(readable_output='', outputs={}, raw_response=contents)
else:
demisto.incidents(incidents)
def fetch_incidents():
last_fetched_event_timestamp = demisto.getLastRun().get('last_fetched_event_timestamp')
if last_fetched_event_timestamp is not None:
last_fetched_event_timestamp = datetime.strptime(last_fetched_event_timestamp, '%Y-%m-%dT%H:%M:%S.%f')
else:
last_fetched_event_timestamp, _ = parse_date_range(FIRST_FETCH_TIMESTAMP)
# Need sometime in the future, so the timestamp will be taken from the query
service_end_date_epoch = int(datetime.now().strftime('%s')) + 1000
global FETCH_QUERY
if 'panw.' in FETCH_QUERY:
service_start_date_epoch = int(last_fetched_event_timestamp.strftime('%s'))
FETCH_QUERY += f' WHERE receive_time>{service_start_date_epoch}'
elif 'tms.' in FETCH_QUERY:
service_start_date_iso = last_fetched_event_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
FETCH_QUERY += f" WHERE serverTime>'{service_start_date_iso}'"
query_data = {
'query': FETCH_QUERY,
'startTime': 0,
'endTime': service_end_date_epoch,
}
response = query_loggings(query_data)
try:
result = response.json()['result']
pages = result['esResult']['hits']['hits']
except ValueError:
raise Exception('Failed to parse the response from Cortex')
incident_pairs = []
max_fetched_event_timestamp = last_fetched_event_timestamp
for page in pages:
incident, time_received = convert_log_to_incident(page)
if 'panw.' in FETCH_QUERY:
time_received_dt = datetime.fromtimestamp(time_received)
elif 'tms.' in FETCH_QUERY:
time_received_dt = datetime.strptime(time_received, '%Y-%m-%dT%H:%M:%S.%fZ')
incident_pairs.append((incident, time_received_dt))
if incident_pairs:
incidents, max_fetched_event_timestamp = process_incident_pairs(incident_pairs, 100) # max 100 per run
demisto.setLastRun({
'last_fetched_event_timestamp': max_fetched_event_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f')
})
demisto.incidents(incidents)
else:
demisto.incidents([])
def fetch_incidents():
params = demisto.params()
user_key = params.get('queryUserKey')
user_key = user_key if user_key else ADMIN_EMAIL
query = '' if params['query'] is None else params['query']
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch = datetime.datetime.now() - datetime.timedelta(days=1)
else:
last_fetch = datetime.datetime.strptime(
last_fetch, '%Y-%m-%dT%H:%M:%SZ')
current_fetch = last_fetch
credentials = get_credentials(
['https://www.googleapis.com/auth/gmail.readonly', ], user_key)
service = discovery.build('gmail', 'v1', credentials=credentials)
query += last_fetch.strftime(' after:%Y/%m/%d')
LOG('GMAIL: fetch parameters:\nuser: %s\nquery=%s\nfetch time: %s' %
(user_key, query, last_fetch, ))
result = service.users().messages().list(
userId=user_key, maxResults=100, q=query).execute()
incidents = []
# so far, so good
LOG('GMAIL: possible new incidents are %s' % (result, ))
for msg in result.get('messages', []):
msg_result = service.users().messages().get(
id=msg['id'], userId=user_key).execute()
incident = mail_to_incident(msg_result, service, user_key)
temp_date = datetime.datetime.strptime(
incident['occurred'], '%Y-%m-%dT%H:%M:%SZ')
# update last run
if temp_date > last_fetch:
last_fetch = temp_date + datetime.timedelta(seconds=1)
# avoid duplication due to weak time query
if temp_date > current_fetch:
incidents.append(incident)
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.setLastRun({'time': last_fetch.isoformat().split('.')[0] + 'Z'})
return incidents
def fetch_incidents(query, id_offset=None, fetch_by_created=None, **_):
last_run = demisto.getLastRun()
demisto.debug(f"last_run: {last_run}" if last_run else 'last_run is empty')
id_offset = last_run.get("idOffset") if (last_run and last_run.get("idOffset")) else id_offset
incidents, max_results = [], 50
if id_offset:
query = f'{query} AND id >= {id_offset}'
if fetch_by_created:
query = f'{query} AND created>-1m'
res = run_query(query, '', max_results)
for ticket in res.get('issues'):
id_offset = max(id_offset, ticket.get("id"))
incidents.append(create_incident_from_ticket(ticket))
demisto.setLastRun({"idOffset": id_offset})
demisto.incidents(incidents)
def fetch_incidents_command():
objects_names_to_fetch = OBJECTS_TO_FETCH
fetch_attachments = FETCH_ATTACHMENTS
max_result = MAX_RESULT
fetch_time = FETCH_TIME
query_string = QUERY_STRING
incidents = fetch_incidents(objects_names_to_fetch, fetch_time, max_result, query_string, fetch_attachments,
real_fetch=True)
if incidents:
last_incident_created_time = incidents[-1].get('CreatedDateTime')
next_created_time_to_fetch = \
(datetime.strptime(last_incident_created_time, DATE_FORMAT) + timedelta(seconds=1)).strftime(DATE_FORMAT)
demisto.setLastRun({
'last_created_time': next_created_time_to_fetch,
'objects_names_to_fetch': objects_names_to_fetch
})
return
def fetch_incidents():
query_id = GetEventQuery()
res = GetIncidentsByOrg(query_id)
known_ids = demisto.getLastRun().get('ids', None)
if known_ids is None or not known_ids:
known_ids = []
incidents = []
for inc in res:
if inc.get('incidentId') not in known_ids:
incidents.append({"name": inc.get('eventName', 'New FortiSIEM Event'), "rawJSON": json.dumps(inc)})
if len(known_ids) >= 1000:
known_ids.pop(0)
known_ids.append(inc.get('incidentId'))
demisto.setLastRun({
'ids': known_ids,
'extended_keys': EXTENDED_KEYS
})
demisto.incidents(incidents)
sys.exit(0)
def fetch_incidents():
incidents = []
last_run = demisto.getLastRun()
if last_run and last_run['time_stamp']:
last_update_time = last_run['time_stamp']
else:
# In first run
last_update_time, _ = parse_date_range(FETCH_TIME, date_format='%Y-%m-%dT%H:%M:%S.%f'[:-3])
max_timestamp = last_update_time
if FETCH_BEHAVIORS:
params = 'behaviors.time_stamp>' + last_update_time
behaviors = fetch_behaviors_request(params)
for behavior in behaviors.get('results'):
incident = behavior_to_incident(behavior)
# 0 corresponds to never triggered
time_stamp = behavior.get('behaviors.time_stamp')[:-5] # comapre time_stamp
if time_stamp > max_timestamp:
max_timestamp = time_stamp
incidents.append(incident)
if FETCH_NOTIFICATIONS:
params = 'notifications.time_stamp>' + last_update_time
notifications = search_notifications_request(params)
for notification in notifications.get('results'):
incident = notifications_to_incidents(notification)
time_stamp = notification.get('notifications.time_stamp')[:-5]
if time_stamp > max_timestamp:
max_timestamp = time_stamp
incidents.append(incident)
demisto.setLastRun({
'time_stamp': max_timestamp
})
demisto.incidents(incidents)
def fetch_incidents():
"""
Fetch events from this integration and return them as Demisto incidents
returns:
Demisto incidents
"""
# demisto.getLastRun() will returns an obj with the previous run in it.
last_run = demisto.getLastRun()
# Get the last fetch time and data if it exists
last_fetch = last_run.get('last_fetched_data_timestamp')
last_fetched_data = last_run.get('last_fetched_data')
# Handle first time fetch, fetch incidents retroactively
if not last_fetch:
last_fetch, _ = parse_date_range(FETCH_TIME, to_timestamp=True)
args = {'rows': MAX_ROWS, 'query': FETCH_QUERY}
column_descriptions, data = snowflake_query(args)
data.sort(key=lambda k: k[DATETIME_COLUMN])
# convert the data/events to demisto incidents
incidents = []
for row in data:
incident = row_to_incident(column_descriptions, row)
incident_timestamp = incident.get('timestamp')
# Update last run and add incident if the incident is newer than last fetch
if incident_timestamp and incident_timestamp >= last_fetch:
last_fetch = incident_timestamp
if incident.get('rawJSON') != last_fetched_data:
last_fetched_data = incident.get('rawJSON')
del incident['timestamp']
incidents.append(incident)
this_run = {
'last_fetched_data': last_fetched_data,
'last_fetched_data_timestamp': last_fetch
}
demisto.setLastRun(this_run)
demisto.incidents(incidents)
def fetch_incidents_command():
"""
Fetch alerts from Canary Tools as incidents in Demisto
last_fetch: The latest fetched alert creation time
"""
last_fetch = demisto.getLastRun().get('time')
if last_fetch is None:
last_fetch = parse_date_range(FETCH_DELTA, '%Y-%m-%d-%H:%M:%S')[0]
# All alerts retrieved from get_alerts are newer than last_fetch and are in a chronological order
alerts = get_alerts(last_fetch)
incidents = []
current_fetch = last_fetch
for alert in alerts:
current_fetch = 1000 * (int(demisto.get(alert, 'description.created')))
current_fetch = timestamp_to_datestring(current_fetch, '%Y-%m-%d-%H:%M:%S')
incident = create_incident(alert)
incidents.append(incident)
demisto.incidents(incidents)
demisto.setLastRun({'time': current_fetch})
def fetch_incidents():
"""
Generates and fetches phishing email-like incidents
Generates phishing email-like incidents, with the number of incidents, the
speed of generation and the recurring time period all set by the values
entered in the integration instance parameters. This method operates
under the assumption that fetch-incidents is called once per minute.
returns:
Email-like incidents
"""
try:
update_parameters()
if not FREQUENCY: # Run once
last_run = 0 if not demisto.getLastRun() else demisto.getLastRun().get('numOfIncidentsCreated', 0)
num_of_incidents_created, incidents = generate_incidents(last_run)
demisto.incidents(incidents)
demisto.setLastRun({'numOfIncidentsCreated': last_run + num_of_incidents_created})
return
else:
minutes_of_generation = MAX_NUM_OF_INCIDENTS / float(INCIDENTS_PER_MINUTE)
if minutes_of_generation > FREQUENCY:
err_msg = 'The maximum number of incidents divided by the incidents to generate per minute'
err_msg += ' exceeds every how often incidents should be generated. Please increase the value'
err_msg += ' entered for how often the integration should generate incidents.'
raise Exception(err_msg)
run_counter = 0 if not demisto.getLastRun() else demisto.getLastRun().get('run_count', 0)
last_run = 0 if not demisto.getLastRun() else demisto.getLastRun().get('numOfIncidentsCreated', 0)
should_run = run_counter % FREQUENCY
if should_run < math.ceil(minutes_of_generation): # then should run
if should_run == 0:
last_run = 0
num_of_incidents_created, incidents = generate_incidents(last_run)
demisto.incidents(incidents)
total_incidents_created = last_run + num_of_incidents_created
updated_run_count = run_counter + 1
demisto.setLastRun({
'numOfIncidentsCreated': total_incidents_created,
'run_count': updated_run_count
})
return
else:
updated_run_count = run_counter + 1
demisto.setLastRun({
'numOfIncidentsCreated': last_run,
'run_count': updated_run_count
})
demisto.incidents([])
except Exception:
raise
def fetch_incidents():
query_params = {}
incidents = []
if FETCH_TIME:
fetch_time = FETCH_TIME
else:
fetch_time = DEFAULTS['fetch_time']
last_run = demisto.getLastRun()
if 'time' not in last_run:
snow_time, _ = parse_date_range(fetch_time, '%Y-%m-%d %H:%M:%S')
else:
snow_time = last_run['time']
query = ''
if SYSPARM_QUERY:
query += SYSPARM_QUERY + '^'
query += 'ORDERBY{0}^{0}>{1}'.format(TIMESTAMP_FIELD, snow_time)
if query:
query_params['sysparm_query'] = query
query_params['sysparm_limit'] = SYSPARM_LIMIT
path = 'table/' + TICKET_TYPE
res = send_request(path, 'get', params=query_params)
count = 0
parsed_snow_time = datetime.strptime(snow_time, '%Y-%m-%d %H:%M:%S')
for result in res.get('result', []):
labels = []
if TIMESTAMP_FIELD not in result:
raise ValueError("The timestamp field [{}]"
" does not exist in the ticket".format(TIMESTAMP_FIELD))
if count > SYSPARM_LIMIT:
break
try:
if datetime.strptime(result[TIMESTAMP_FIELD], '%Y-%m-%d %H:%M:%S') < parsed_snow_time:
continue
except Exception:
pass
for k, v in result.iteritems():
if isinstance(v, basestring):
labels.append({
'type': k,
'value': v
})
else:
labels.append({
'type': k,
'value': json.dumps(v)
})
severity = SEVERITY_MAP.get(result.get('severity', ''), 0)
file_names = []
if GET_ATTACHMENTS:
file_entries = get_ticket_attachment_entries(result['sys_id'])
for file_result in file_entries:
if file_result['Type'] == entryTypes['error']:
raise Exception('Error getting attachment: ' + str(file_result['Contents']))
file_names.append({
'path': file_result['FileID'],
'name': file_result['File']
})
incidents.append({
'name': 'ServiceNow Incident ' + result.get('number'),
'labels': labels,
'details': json.dumps(result),
'severity': severity,
'attachment': file_names,
'rawJSON': json.dumps(result)
})
count += 1
snow_time = result[TIMESTAMP_FIELD]
demisto.incidents(incidents)
demisto.setLastRun({'time': snow_time})
