def fetch_incidents_command(): """ Fetch alerts from Canary Tools as incidents in Demisto last_fetch: The latest fetched alert creation time """ last_fetch = demisto.getLastRun().get('time') if last_fetch is None: last_fetch = parse_date_range(FETCH_DELTA, '%Y-%m-%d-%H:%M:%S')[0] # All alerts retrieved from get_alerts are newer than last_fetch and are in a chronological order alerts = get_alerts(last_fetch) incidents = [] current_fetch = last_fetch for alert in alerts: current_fetch = 1000 * ( int(demisto.get(alert, 'description.created')) + 1) current_fetch = timestamp_to_datestring(current_fetch, '%Y-%m-%d-%H:%M:%S') incident = create_incident(alert) incidents.append(incident) demisto.incidents(incidents) demisto.setLastRun({'time': current_fetch})
def fetch_indicators_command(client, feed_type, src_val, src_type, default_type, last_fetch): """Implements fetch-indicators command""" last_fetch_timestamp = get_last_fetch_timestamp(last_fetch, client.time_method, client.fetch_time) now = datetime.now() ioc_lst: list = [] ioc_enrch_lst: list = [] if FEED_TYPE_GENERIC not in feed_type: # Insight is the name of the indicator object as it's saved into the database search = get_scan_insight_format(client, now, last_fetch_timestamp, feed_type) for hit in search.scan(): hit_lst, hit_enrch_lst = extract_indicators_from_insight_hit(hit) ioc_lst.extend(hit_lst) ioc_enrch_lst.extend(hit_enrch_lst) else: search = get_scan_generic_format(client, now, last_fetch_timestamp) for hit in search.scan(): ioc_lst.extend( extract_indicators_from_generic_hit(hit, src_val, src_type, default_type)) if ioc_lst: for b in batch(ioc_lst, batch_size=2000): demisto.createIndicators(b) if ioc_enrch_lst: ioc_enrch_batches = create_enrichment_batches(ioc_enrch_lst) for enrch_batch in ioc_enrch_batches: # ensure batch sizes don't exceed 2000 for b in batch(enrch_batch, batch_size=2000): demisto.createIndicators(b) demisto.setLastRun({'time': now.timestamp() * 1000})
def main(): # pragma: no cover params = demisto.params() base_url = urljoin(params.get('url'), '/api/v2/') verify_ssl = not params.get('insecure', False) proxy = params.get('proxy') client = Client(base_url=base_url, verify=verify_ssl, proxy=proxy) command = demisto.command() demisto.debug(f'Command being called is {command}') # Switch case commands = { 'test-module': test_module_command, 'fetch-incidents': fetch_incidents_command, f'{INTEGRATION_COMMAND_NAME}-list-events': list_events_command, f'{INTEGRATION_COMMAND_NAME}-get-event': get_event_command, f'{INTEGRATION_COMMAND_NAME}-delete-event': close_event_command, f'{INTEGRATION_COMMAND_NAME}-update-event': update_event_command, f'{INTEGRATION_COMMAND_NAME}-create-event': create_event_command, f'{INTEGRATION_COMMAND_NAME}-query': query_command } try: if command == 'fetch-incidents': incidents, new_last_run = fetch_incidents_command( client, last_run=demisto.getLastRun()) demisto.incidents(incidents) demisto.setLastRun(new_last_run) elif command in commands: readable_output, outputs, raw_response = commands[command]( client, demisto.args()) return_outputs(readable_output, outputs, raw_response) # Log exceptions except Exception as e: err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]' return_error(err_msg, error=e)
def main(): """ PARSE AND VALIDATE INTEGRATION PARAMS """ service_principal = demisto.params().get('credentials').get('identifier') secret = demisto.params().get('credentials').get('password') # Remove trailing slash to prevent wrong URL path to service server_url = demisto.params()['url'][:-1] \ if (demisto.params()['url'] and demisto.params()['url'].endswith('/')) else demisto.params()['url'] api_version = demisto.params().get('api_version') verify_certificate = not demisto.params().get('insecure', False) # How many time before the first fetch to retrieve incidents fetch_time = demisto.params().get('fetch_time', '3 days') threat_status = argToList(demisto.params().get('threat_status')) threat_type = argToList(demisto.params().get('threat_type')) event_type_filter = demisto.params().get('events_type') fetch_limit = 50 # Remove proxy if not set to true in params proxies = handle_proxy() LOG('Command being called is %s' % (demisto.command())) try: client = Client(server_url, api_version, verify_certificate, service_principal, secret, proxies) if demisto.command() == 'test-module': results = test_module(client, fetch_time, event_type_filter) return_outputs(results, None) elif demisto.command() == 'fetch-incidents': integration_context = demisto.getIntegrationContext() next_run, incidents, remained_incidents = fetch_incidents( client=client, last_run=demisto.getLastRun(), first_fetch_time=fetch_time, event_type_filter=event_type_filter, threat_status=threat_status, threat_type=threat_type, limit=fetch_limit, integration_context=integration_context) # Save last_run, incidents, remained incidents into integration demisto.setLastRun(next_run) demisto.incidents(incidents) # preserve context dict integration_context['incidents'] = remained_incidents demisto.setIntegrationContext(integration_context) elif demisto.command() == 'proofpoint-get-events': return_outputs(*get_events_command(client, demisto.args())) except Exception as e: return_error(str(e))
def main(): params = demisto.params() client = Client(base_url=urljoin(params.get('host'), '/siem/v1/configs'), verify=not params.get('insecure', False), proxy=params.get('proxy'), auth=EdgeGridAuth( client_token=params.get('clientToken'), access_token=params.get('accessToken'), client_secret=params.get('clientSecret'))) commands = { f"test-module": test_module_command, f"{INTEGRATION_COMMAND_NAME}-get-events": get_events_command } command = demisto.command() demisto.debug(f'Command being called is {command}') try: if command == 'fetch-incidents': incidents, new_last_run = fetch_incidents_command( client, fetch_time=params.get('fetchTime'), fetch_limit=params.get('fetchLimit'), config_ids=params.get('configIds'), last_run=demisto.getLastRun().get('lastRun')) demisto.incidents(incidents) demisto.setLastRun(new_last_run) else: human_readable, entry_context, raw_response = commands[command]( client, **demisto.args()) return_outputs(human_readable, entry_context, raw_response) except Exception as e: err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]' return_error(err_msg, error=e)
def update_indicators_iterator(indicators_iterator: List[Dict[str, Any]], params_dict: Dict[str, Any], is_fetch: bool) -> Optional[List[Dict[str, Any]]]: """ sorts the indicators by their timestamp and returns a list of only new indicators received from MISP Args: params_dict: user's params sent to misp indicators_iterator: list of indicators is_fetch: flag for wether funciton was called for fetching command or a get Returns: Sorted list of new indicators """ last_run = demisto.getLastRun() indicators_iterator.sort(key=lambda indicator: indicator['value']['timestamp']) if last_run is None: return indicators_iterator if params_dict != last_run.get('params'): if is_fetch: demisto.setLastRun(None) return indicators_iterator last_timestamp = int(last_run.get('timestamp')) for index in range(len(indicators_iterator)): if int(indicators_iterator[index]['value']['timestamp']) > last_timestamp: return indicators_iterator[index:] return []
def fetch_incidents(): param_dict = {} now_time = datetime.utcnow() now = datetime.isoformat(now_time) lastRunObject = demisto.getLastRun() if lastRunObject: param_dict['since'] = lastRunObject['time'] else: param_dict['since'] = datetime.isoformat(now_time - timedelta(minutes=int(FETCH_INTERVAL))) param_dict['until'] = now url = SERVER_URL + GET_INCIDENTS_SUFFIX + configure_status() res = http_request('GET', url, param_dict) _, parsed_incidents, raw_responses = parse_incident_data(res.get('incidents', [])) incidents = [] for incident, raw_response in zip(parsed_incidents, raw_responses): incidents.append({ 'name': incident['ID'] + ' - ' + incident['Title'], 'occurred': incident['created_at'], 'severity': translate_severity(incident['urgency']), 'rawJSON': json.dumps(raw_response) }) demisto.incidents(incidents) demisto.setLastRun({'time': now})
def fetch_incidents(client: Client, first_fetch_time, fetch_limit, fetch_state, fetch_severity, fetch_status, fetch_app_ids): last_run = demisto.getLastRun() last_fetch = last_run.get('last_run_time') if last_fetch is None: last_fetch = dateparser.parse(first_fetch_time, settings={'TIMEZONE': 'UTC'}) last_fetch = last_fetch.strftime( SAAS_SECURITY_DATE_FORMAT )[:-4] + 'Z' # format ex: 2021-08-23T09:26:25.872Z current_fetch = last_fetch results = client.get_incidents(limit=fetch_limit, from_time=last_fetch, state=fetch_state, severity=fetch_severity, status=fetch_status, app_ids=fetch_app_ids).get('resources', []) incidents = list() for inc in results: incident = convert_to_xsoar_incident(inc) incidents.append(incident) date_updated = inc.get('updated_at') date_updated_dt = datetime.strptime(date_updated, SAAS_SECURITY_DATE_FORMAT) if date_updated_dt > datetime.strptime(current_fetch, SAAS_SECURITY_DATE_FORMAT): current_fetch = date_updated demisto.setLastRun({'last_run_time': current_fetch}) demisto.incidents(incidents)
def fetch_incidents(): last_run = demisto.getLastRun() last_fetch = last_run.get('time') # handle first time fetch if last_fetch is None: last_fetch = datetime.now() - timedelta(days=5) else: last_fetch = datetime.strptime(last_fetch, '%Y-%m-%dT%H:%M:%SZ') LOG('iterating on detections, looking for more recent than {}'.format( last_fetch)) incidents = [] for raw_detection in get_unacknowledge_detections(last_fetch, per_page=2): LOG('found detection #{}'.format(raw_detection['id'])) incident = detection_to_incident(raw_detection) incidents.append(incident) if len(incidents) != 0: last_fetch = max([ get_time_obj(incident['occurred']) for incident in incidents ]) # noqa:F812 demisto.setLastRun( {'time': get_time_str(last_fetch + timedelta(seconds=1))}) demisto.incidents(incidents)
def get_no_update_value(response: requests.models.Response, url: str) -> bool: """ detect if the feed response has been modified according to the headers etag and last_modified. For more information, see this: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag Args: response: (requests.Response) The feed response. url: (str) The feed URL (service). Returns: boolean with the value for noUpdate argument. The value should be False if the response was modified. """ if response.status_code == 304: demisto.debug('No new indicators fetched, createIndicators will be executed with noUpdate=True.') return True etag = response.headers.get('ETag') last_modified = response.headers.get('Last-Modified') if not etag and not last_modified: demisto.debug('Last-Modified and Etag headers are not exists,' 'createIndicators will be executed with noUpdate=False.') return False last_run = demisto.getLastRun() last_run[url] = {'last_modified': last_modified, 'etag': etag} demisto.setLastRun(last_run) demisto.debug('New indicators fetched - the Last-Modified value has been updated,' ' createIndicators will be executed with noUpdate=False.') return False
def main(command: str, demisto_params: dict): demisto.debug(f'Command being called is {command}') try: demisto_params['client_secret'] = demisto_params['credentials'][ 'password'] push_to_xsiam = argToBoolean( demisto_params.get('should_push_events', 'false')) vendor = demisto_params.get('vendor') product = demisto_params.get('product') after = demisto_params.get('after') if after and not isinstance(after, int): timestamp = dateparser.parse(after) after = int(timestamp.timestamp() * 1000) # type: ignore options = IntegrationOptions.parse_obj(demisto_params) request = DefenderHTTPRequest.parse_obj(demisto_params) authenticator = DefenderAuthenticator.parse_obj(demisto_params) client = DefenderClient(request=request, options=options, authenticator=authenticator, after=after) get_events = DefenderGetEvents(client=client, options=options) if command == 'test-module': return_results(test_module(get_events=get_events)) elif command in ('fetch-events', 'microsoft-defender-cloud-apps-get-events'): events = get_events.run() if command == 'fetch-events': # publishing events to XSIAM send_events_to_xsiam(events, vendor=vendor, product=product) demisto.setLastRun(DefenderGetEvents.get_last_run(events)) elif command == 'microsoft-defender-cloud-apps-get-events': command_results = CommandResults( readable_output=tableToMarkdown( 'microsoft defender cloud apps events', events, headerTransform=pascalToSpace), outputs_prefix='Microsoft.Events', outputs_key_field='_id', outputs=events, raw_response=events, ) return_results(command_results) if push_to_xsiam: # publishing events to XSIAM send_events_to_xsiam(events, vendor=vendor, product=product) # Log exceptions and return errors except Exception as e: demisto.error(traceback.format_exc()) # print the traceback return_error(f'Failed to execute {command} command.\nError:\n{str(e)}')
def fetch_incidents(): last_run = demisto.getLastRun() if last_run and last_run['last_alert_fetched_time']: last_alert_fetched_time = datetime.strptime(last_run['last_alert_fetched_time'], '%Y-%m-%dT%H:%M:%S.%f') else: last_alert_fetched_time = datetime.now() - timedelta(days=300) latest_creation_time = last_alert_fetched_time alerts = list_alerts()['value'] incidents = [] for alert in alerts: # Removing 'Z' from timestamp and converting to datetime alert_creation_time = datetime.strptime(alert['alertCreationTime'][:-2], '%Y-%m-%dT%H:%M:%S.%f') alert_status = alert['status'] alert_severity = alert['severity'] if alert_creation_time > last_alert_fetched_time and alert_status in FETCH_STATUS and \ alert_severity in FETCH_SEVERITY: incident = alert_to_incident(alert) incidents.append(incident) if alert_creation_time > latest_creation_time: latest_creation_time = alert_creation_time demisto.setLastRun({ 'last_alert_fetched_time': datetime.strftime(latest_creation_time, '%Y-%m-%dT%H:%M:%S.%f') }) demisto.incidents(incidents)
def fetch_incidents(): last_run = demisto.getLastRun() # Get the last fetch time, if exists last_fetch = last_run.get('time') # Handle first time fetch, fetch incidents retroactively if last_fetch is None: last_fetch, _ = parse_date_range(FETCH_TIME, date_format='%Y-%m-%dT%H:%M:%S') latest = datetime.strptime(last_fetch, '%Y-%m-%dT%H:%M:%S') demisto.debug('getting alarms since {}'.format(last_fetch)) incidents = [] items = list_alerts(time_frame='Custom', start_time=last_fetch) demisto.debug('got {} new alarms'.format(len(items))) for item in items: incident_date = datetime.strptime(item['ALERT_TIME'], '%Y-%m-%d %H:%M:%S') incident = { 'Type': 'Fidelis', 'name': '{} {}'.format(item['ALERT_ID'], item['SUMMARY']), 'occurred': incident_date.strftime('%Y-%m-%dT%H:%M:%SZ'), 'rawJSON': json.dumps(item), } latest = max(latest, incident_date) incidents.append(incident) if latest != last_fetch: last_fetch = (latest + timedelta(seconds=1)).strftime('%Y-%m-%dT%H:%M:%S') demisto.setLastRun({'time': last_fetch}) demisto.incidents(incidents)
def fetch_incidents_command(client, params, is_test_module=False): list_services = params.get('service') list_event_type = params.get('event_type') first_fetch_time = parse_date_to_isoformat( params.get('first_fetch', '3 days'), 'first_fetch_time') max_results = int(params.get('max_fetch', 50)) if not max_results or max_results > MAX_INCIDENTS_TO_FETCH: max_results = MAX_INCIDENTS_TO_FETCH last_run = demisto.getLastRun() # getLastRun() gets the last run dict if not last_run: last_run = creates_empty_dictionary_of_last_run( list_services, list_event_type) next_run, incidents = fetch_incidents( client=client, max_results=max_results, last_run=last_run, # getLastRun() gets the last run dict list_services=list_services, list_event_type=list_event_type, first_fetch_time=first_fetch_time, is_test_module=is_test_module) if is_test_module: return demisto.setLastRun(next_run) demisto.incidents(incidents)
def fetch_incidents(self, params: Dict): last_run = demisto.getLastRun() current_run = {} incidents = [] if params.get('fetchType', 'alarms') in ('alarms', 'both'): start_time = last_run.get('alarms', {}).get( 'time', parse_date_range(params.get('fetchTime'), self.demisto_format)[0]) start_id = int( last_run.get('alarms', {}).get('id', params.get('startingFetchID'))) temp_incidents, current_run['alarms'] = \ self.__alarms_to_incidents(start_time, start_id, int(params.get('fetchLimitAlarms', 5))) incidents.extend(temp_incidents) if params.get('fetchType') in ('cases', 'both'): start_id = int( last_run.get('cases', {}).get('id', params.get('startingFetchID'))) temp_incidents, current_run['cases'] = \ self.__cases_to_incidents(start_id=start_id, limit=int(params.get('fetchLimitCases', 5))) incidents.extend(temp_incidents) demisto.setLastRun(current_run) demisto.incidents(incidents)
def fetch_incidents(client, alert_type): """ fetches alerts and turns them into incidents. """ last_run = demisto.getLastRun() # Get the last fetch time, if exists last_fetch = last_run.get('time') # Handle first time fetch, fetch incidents retroactively if last_fetch is None: fetch_time, _ = parse_date_range(FETCH_TIME) last_fetch = fetch_time.strftime(DATE_FORMAT) incidents = [] ### items = iter_all_alerts(client, alert_type, last_fetch, datetime.now().strftime(DATE_FORMAT), severity=SEVERITY_TYPE) ### for item in items: incident = item_to_incident(item) incident_date = datetime.strptime(incident['timestamp'], DATE_FORMAT) # Update last run and add incident if the incident is newer than last fetch if incident_date.timestamp() > datetime.strptime( last_fetch, DATE_FORMAT).timestamp(): last_fetch = incident_date.strftime(DATE_FORMAT) incidents.append(incident) demisto.setLastRun({'time': last_fetch}) demisto.incidents(incidents)
def fetch_incidents(client: Client, fetch_time: str, fetch_shaping: str, last_run: Dict, fetch_limit: str, fetch_queue_id: Optional[list] = None, fetch_filter: Optional[str] = None) -> list: """ This function will execute each interval (default is 1 minute). Args: client (Client): Quest Kace Client fetch_time: time interval for fetch incidents. fetch_shaping: shaping for the request. fetch_filter: custom filters for the request. fetch_limit: limit for number of fetch incidents per fetch. fetch_queue_id: queue id for fetch, if not given then fetch runs on all tickets in the system last_run (dateparser.time): The greatest incident created_time we fetched from last fetch Returns: incidents: Incidents that will be created in Demisto """ if not fetch_queue_id or fetch_queue_id[0] == 'All': fetch_queue_id = get_queue_ids(client) time_format = '%Y-%m-%dT%H:%M:%SZ' if not last_run: # if first time running new_last_run = { 'last_fetch': parse_date_range(fetch_time, date_format=time_format)[0] } else: new_last_run = last_run if not fetch_shaping: fetch_shaping = shaping_fetch(client, fetch_queue_id) parsed_last_time = datetime.strptime(new_last_run.get('last_fetch', ''), time_format) fetch_filter_for_query = f'created gt {parsed_last_time}' if fetch_queue_id: queue_id_str = ';'.join(fetch_queue_id) filter_by_queue_id = f'hd_queue_id in {queue_id_str}' fetch_filter_for_query = f'{fetch_filter_for_query},{filter_by_queue_id}' if fetch_filter: fetch_filter_for_query = f'{fetch_filter_for_query},{fetch_filter}' demisto.info(f"Fetching Incident has Started,\n" f"Fetch filter is {fetch_filter_for_query}\n" f"Last fetch was on {str(parsed_last_time)}") client.update_token() items: dict = client.tickets_list_request(fetch_shaping, fetch_filter_for_query) items: list = items.get('Tickets', []) incidents, last_incident_time = parse_incidents(items, fetch_limit, time_format, parsed_last_time) last_incident_time = last_incident_time.strftime(time_format) demisto.info(f"Fetching Incident has Finished\n" f"Fetch limit was {fetch_limit}" f"Last fetch was on {str(last_incident_time)}\n" f"Number of incidents was {len(incidents)}") demisto.setLastRun({'last_fetch': last_incident_time}) return incidents
def main(): # Write configure here params = {k: v for k, v in demisto.params().items() if v is not None} handle_proxy() client = TAXIIClient(**params) command = demisto.command() demisto.info('Command being called is {command}'.format(command=command)) # Switch case commands = { 'test-module': test_module, 'get-indicators': get_indicators_command } try: if demisto.command() == 'fetch-indicators': indicators = fetch_indicators_command(client) # we submit the indicators in batches for b in batch(indicators, batch_size=2000): demisto.createIndicators(b) demisto.setLastRun({'time': client.last_taxii_run}) else: readable_output, outputs, raw_response = commands[command]( client, demisto.args()) return_outputs(readable_output, outputs, raw_response) except Exception as e: err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]' raise Exception(err_msg)
def fetch_incidents(client, params): now = datetime.utcnow() incidents = [] fetch_params = _set_fetch_params(params, now) fetch_passwords = params.get('fetch_passwords') severity = getattr(IncidentSeverity, params['fetch_severity'].upper()) incident_type = params.get('incidentType') if not incident_type or incident_type == '': incident_type = INCIDENT_TYPE response = client.search('exposures', fetch_params) if response and response.get('data'): data = _process_exposure_data(response['data'], fetch_passwords) i = { 'name': INCIDENT_NAME, 'occurred': now.strftime(ISO8601_FORMAT), 'rawJSON': json.dumps(data), 'type': incident_type, 'severity': severity } incidents.append(i) demisto.setLastRun({'start_time': now.timestamp()}) return incidents
def fetch_incidents_command(): last_run = demisto.getLastRun() if last_run and 'start_time' in last_run: start_time = datetime.strptime(last_run.get('start_time'), '%Y-%m-%dT%H:%M:%SZ') else: start_time = datetime.now() - timedelta(days=int(FETCH_TIME)) last_time = start_time issue_list = http_request(method='GET', url_suffix=ISSUE_SUFFIX, params={'state': 'all'}) incidents = [] for issue in issue_list: updated_at_str = issue.get('created_at') updated_at = datetime.strptime(updated_at_str, '%Y-%m-%dT%H:%M:%SZ') if updated_at > start_time: inc = { 'name': issue.get('url'), 'occurred': updated_at_str, 'rawJSON': json.dumps(issue) } incidents.append(inc) if updated_at > last_time: last_time = updated_at demisto.setLastRun({'start_time': datetime.strftime(last_time, '%Y-%m-%dT%H:%M:%SZ')}) demisto.incidents(incidents)
def fetch_incidents(query, id_offset, fetch_by_created=None, **_): last_run = demisto.getLastRun() demisto.debug(f"last_run: {last_run}" if last_run else 'last_run is empty') if last_run and last_run.get("idOffset"): id_offset = last_run.get("idOffset") if not id_offset: id_offset = 0 incidents, max_results = [], 50 if id_offset: query = f'{query} AND id >= {id_offset}' if fetch_by_created: query = f'{query} AND created>-1m' res = run_query(query, '', max_results) curr_id = id_offset for ticket in res.get('issues'): ticket_id = int(ticket.get("id")) if ticket_id == curr_id: continue id_offset = max(int(id_offset), ticket_id) incidents.append(create_incident_from_ticket(ticket)) demisto.setLastRun({"idOffset": id_offset}) demisto.incidents(incidents)
def main(): params = demisto.params() username = params.get('credentials').get('identifier') password = params.get('credentials').get('password') server = urljoin(params.get('url'), '/api/v2.0/') # Should we use SSL use_ssl = not params.get('insecure') == 'true' use_proxy = params.get('proxy') == 'true' # Headers to be sent in requests headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', } client = Client(server, use_ssl, use_proxy, headers=headers, auth=(username, password)) # Commands switch case command = demisto.command() commands = {f'{INTEGRATION_COMMAND_NAME}-query': query_command} if command == 'fetch-incidents': # How many time before the first fetch to retrieve incidents fetch_time = params.get('fetch_time', '3 days') columns = params.get('columns') table_name = params.get('table_name') date_name = params.get('date_name') last_run, incidents = fetch_incidents_command(client, demisto.getLastRun(), fetch_time, table_name, columns, date_name) demisto.setLastRun(last_run) demisto.incidents(incidents) elif command in commands: return_outputs(*commands[command](client, demisto.args()))
def fetch_incidents(service): last_run = demisto.getLastRun() and demisto.getLastRun()['time'] search_offset = demisto.getLastRun().get('offset', 0) incidents = [] current_time_for_fetch = datetime.utcnow() dem_params = demisto.params() if demisto.get(dem_params, 'timezone'): timezone = dem_params['timezone'] current_time_for_fetch = current_time_for_fetch + timedelta( minutes=int(timezone)) now = current_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) if demisto.get(dem_params, 'useSplunkTime'): now = get_current_splunk_time(service) current_time_in_splunk = datetime.strptime(now, SPLUNK_TIME_FORMAT) current_time_for_fetch = current_time_in_splunk if len(last_run) == 0: fetch_time_in_minutes = parse_time_to_minutes() start_time_for_fetch = current_time_for_fetch - timedelta( minutes=fetch_time_in_minutes) last_run = start_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) earliest_fetch_time_fieldname = dem_params.get( "earliest_fetch_time_fieldname", "index_earliest") latest_fetch_time_fieldname = dem_params.get("latest_fetch_time_fieldname", "index_latest") kwargs_oneshot = { earliest_fetch_time_fieldname: last_run, latest_fetch_time_fieldname: now, "count": FETCH_LIMIT, 'offset': search_offset } searchquery_oneshot = dem_params['fetchQuery'] if demisto.get(dem_params, 'extractFields'): extractFields = dem_params['extractFields'] extra_raw_arr = extractFields.split(',') for field in extra_raw_arr: field_trimmed = field.strip() searchquery_oneshot = searchquery_oneshot + ' | eval ' + field_trimmed + '=' + field_trimmed oneshotsearch_results = service.jobs.oneshot( searchquery_oneshot, **kwargs_oneshot) # type: ignore reader = results.ResultsReader(oneshotsearch_results) for item in reader: inc = notable_to_incident(item) incidents.append(inc) demisto.incidents(incidents) if len(incidents) < FETCH_LIMIT: demisto.setLastRun({'time': now, 'offset': 0}) else: demisto.setLastRun({ 'time': last_run, 'offset': search_offset + FETCH_LIMIT })
def fetch_incidents(): query = demisto.params().get('query') last_run = demisto.getLastRun() offense_id = last_run['id'] if last_run and 'id' in last_run else 0 if last_run and offense_id == 0: start_time = last_run['startTime'] if 'startTime' in last_run else '0' fetch_query = 'start_time>{0}{1}'.format( start_time, ' AND ({0})'.format(query) if query else '') else: fetch_query = 'id>{0} {1}'.format( offense_id, 'AND ({0})'.format(query) if query else '') # qradar returns offenses sorted desc on id and there's no way to change sorting. # if we get `offensesPerCall` offenses it means we (probably) have more than that so we # start looking for the end of the list by doubling the page position until we're empty. # then start binary search back until you find the end of the list and finally return # `offensesPerCall` from the end. raw_offenses = get_offenses(_range='0-{0}'.format(OFFENSES_PER_CALL), _filter=fetch_query) if len(raw_offenses) >= OFFENSES_PER_CALL: last_offense_pos = find_last_page_pos(fetch_query) raw_offenses = get_offenses(_range='{0}-{1}'.format( last_offense_pos - OFFENSES_PER_CALL + 1, last_offense_pos), _filter=fetch_query) raw_offenses = unicode_to_str_recur(raw_offenses) incidents = [] enrich_offense_res_with_source_and_destination_address(raw_offenses) for offense in raw_offenses: offense_id = max(offense_id, offense['id']) incidents.append(create_incident_from_offense(offense)) demisto.setLastRun({'id': offense_id}) return incidents
def fetch_incidents(): """ Fetches incidents """ incidents = list() # Check for topic in Kafka if TOPIC in KAFKA_CLIENT.topics: kafka_topic = KAFKA_CLIENT.topics[TOPIC] offset, partition = check_params(kafka_topic, old_offset=OFFSET, old_partition=PARTITION) last_offset = demisto.getLastRun().get('last_offset') last_fetch = last_offset if last_offset > offset else offset # If need to fetch latest_offset = check_latest_offset(kafka_topic, partition) if latest_offset > last_fetch: consumer = kafka_topic.get_simple_consumer( auto_offset_reset=last_fetch, reset_offset_on_start=True) for i in range(last_fetch, latest_offset): """ consumer.consume() will consume only one message from the given offset. """ message = consumer.consume() incidents.append(create_incident(message=message, topic=TOPIC)) if message.offset == latest_offset: break demisto.setLastRun({'last_offset': latest_offset}) demisto.incidents(incidents) else: return_error( 'No such topic \'{}\' to fetch incidents from.'.format(TOPIC))
def fetch_incidents(): last_run = demisto.getLastRun() if last_run and 'time' in last_run: last_fetch = last_run.get('time') last_fetch = datetime.strptime(last_fetch, TIME_FORMAT) else: last_fetch = parse_date_range( demisto.params().get('fetch_time', '3 days'), TIME_FORMAT)[0] LOG('iterating on detections, looking for more recent than {}'.format( last_fetch)) incidents = [] for raw_detection in get_unacknowledged_detections(last_fetch, per_page=2): LOG('found detection #{}'.format(raw_detection['id'])) incident = detection_to_incident(raw_detection) incidents.append(incident) if incidents: last_fetch = max([ get_time_obj(incident['occurred']) for incident in incidents ]) # noqa:F812 demisto.setLastRun( {'time': get_time_str(last_fetch + timedelta(seconds=1))}) demisto.incidents(incidents)
def fetch_incidents( client: Client, first_fetch: str, dashboard_id: str = None, panel_id: str = None, alert_name: str = None, state: str = None, limit: Optional[int] = MAX_INCIDENTS_TO_FETCH) -> List[dict]: limit = limit or MAX_INCIDENTS_TO_FETCH last_fetch = demisto.getLastRun().get('last_fetch') last_id_fetched = demisto.getLastRun().get('last_id_fetched', -1) fetch_start_time = calculate_fetch_start_time(last_fetch, first_fetch) demisto.debug( f'last fetch was at: {last_fetch}, last id fetched was: {last_id_fetched}, ' f'time to fetch from is: {fetch_start_time}') alerts = client.alerts_list_request(dashboard_id=argToList(dashboard_id), panel_id=panel_id, query=alert_name, state=argToList(state)) last_fetch, last_id_fetched, incidents = parse_alerts( alerts, limit, fetch_start_time, last_id_fetched) demisto.debug( f'last fetch now is: {last_fetch}, last id fetched is now: {last_id_fetched}, ' f'number of incidents fetched is {len(incidents)}, number of alerts got is {len(alerts)}' ) demisto.setLastRun({ 'last_fetch': str(date_to_timestamp(last_fetch, DATE_FORMAT)), 'last_id_fetched': last_id_fetched }) return incidents
def fetch_incidents(): param_dict = {} now_time = datetime.utcnow() now = datetime.isoformat(now_time) lastRunObject = demisto.getLastRun() if lastRunObject: param_dict['since'] = lastRunObject['time'] else: param_dict['since'] = datetime.isoformat(now_time - timedelta( minutes=int(FETCH_INTERVAL))) param_dict['until'] = now url = SERVER_URL + GET_INCIDENTS_SUFFIX + configure_status() res = http_request('GET', url, param_dict) _, parsed_incidents, raw_responses = parse_incident_data( res.get('incidents', [])) incidents = [] for incident, raw_response in zip(parsed_incidents, raw_responses): incidents.append({ 'name': incident['ID'] + ' - ' + incident['Title'], 'occurred': incident['created_at'], 'severity': translate_severity(incident['urgency']), 'rawJSON': json.dumps(raw_response) }) demisto.incidents(incidents) demisto.setLastRun({'time': now})
def fetch_incidents(client: Client): last_run = demisto.getLastRun() last_timestamp = int(last_run.get('timestamp', 0)) res = client.get_cases() res[:] = [ x for x in res if x['createdAt'] > last_timestamp and x['status'] == 'Open' ] res = sorted(res, key=lambda x: x['createdAt']) incidents = list() instance_name = demisto.integrationInstance() mirror_direction = demisto.params().get('mirror') mirror_direction = None if mirror_direction == "Disabled" else mirror_direction for case in res: case['dbotMirrorDirection'] = mirror_direction case['dbotMirrorInstance'] = instance_name incident = { 'name': case['title'], 'occurred': timestamp_to_datestring(case['createdAt']), 'severity': case['severity'], 'rawJSON': json.dumps(case) } incidents.append(incident) last_timestamp = case['createdAt'] if case[ 'createdAt'] > last_timestamp else last_timestamp demisto.setLastRun({"timestamp": str(last_timestamp)}) return incidents
def fetch_incidents(): date_format = '%Y-%m-%dT%H:%M:%S' last_run = demisto.getLastRun() if last_run and last_run.get('last_fetched_event_timestamp'): last_update_time = last_run['last_fetched_event_timestamp'] else: last_update_time = parse_date_range(FETCH_TIME, date_format=date_format)[0] incidents = [] limit: int = int(demisto.params().get('fetch_limit', '')) response_content = list_alerts({'sort_direction': 'asc', 'limit': limit, 'last_modified_min_date': last_update_time}) alerts: List = response_content.get('alerts', []) if alerts: for alert in alerts: incident = alert_to_incident(alert) incidents.append(incident) # max_update_time is the timestamp of the last alert in alerts (alerts is a sorted list) last_alert_timestamp: str = str(alerts[-1].get('timestamp', '')) # ZeroFox is using full timestamp ISO 8061 format which includes the GMT field i.e (+00:00/-00:00) # The option to refine the search of alerts in the API is by the ISO 8061 format without the GMT field. if '+' in last_alert_timestamp: max_update_time = last_alert_timestamp.split('+')[0] else: max_update_time = last_alert_timestamp.split('-')[0] # add 1 second to last alert timestamp, in order to prevent duplicated alerts max_update_time = (datetime.strptime(max_update_time, date_format) + timedelta(seconds=1)).isoformat() demisto.setLastRun({'last_fetched_event_timestamp': max_update_time}) demisto.incidents(incidents)
def fetch_incidents(client, first_fetch, max_fetch): """ This function retrieves new incidents every interval (default is 1 minute). """ now = datetime.timestamp(datetime.utcnow()) last_run_object = demisto.getLastRun() last_run = last_run_object.get('time', None) if last_run_object else None if not last_run: if first_fetch: last_run = float(first_fetch) else: last_run = datetime.timestamp(datetime.utcnow() - timedelta(days=1)) result = client.get_incidents(last_run, now) if not result.get('success'): raise DemistoException(f"ERROR: {result.get('message')}; last_run: {last_run}; now: {now}") lp_incidents = result.get('incidents') incidents = [] if len(lp_incidents) > max_fetch: next_fetch_time = lp_incidents[max_fetch]['detection_timestamp'] lp_incidents = lp_incidents[:max_fetch] else: next_fetch_time = now demisto.info(f"Executing LogPoint fetch_incidents between {last_run} and {next_fetch_time} Timestamp.") for inc in lp_incidents: detection_ts = inc['detection_timestamp'] dt = datetime.utcfromtimestamp(detection_ts) occurred = dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ') incidents.append({ 'name': inc.get('name', 'LogPoint - No name'), 'occurred': occurred, 'severity': get_demisto_severity(inc.get('risk_level')), 'rawJSON': json.dumps(inc) }) demisto.setLastRun({'time': next_fetch_time}) return incidents
def fetch(): """ Query viewer should be defined in ArcSight ESM. fetch incidents fetches the results of query viewer and converts them to Demisto incidents. We can query Cases or Events. If Cases are fetched then the query viewer query must return fields ID and Create Time. If Events are fetched then Event ID and Start Time. """ events_query_viewer_id = demisto.params().get('viewerId') cases_query_viewer_id = demisto.params().get('casesQueryViewerId') type_of_incident = 'case' if events_query_viewer_id else 'event' last_run = json.loads(demisto.getLastRun().get('value', '{}')) already_fetched = last_run.get('already_fetched', []) fields, query_results = get_query_viewer_results(events_query_viewer_id or cases_query_viewer_id) # sort query_results by creation time query_results.sort(key=lambda k: int(k.get('Start Time') or k.get('Create Time'))) incidents = [] for result in query_results: # convert case or event to demisto incident r_id = result.get('ID') or result.get('Event ID') if r_id not in already_fetched: create_time_epoch = int(result.get('Start Time') or result.get('Create Time')) result['Create Time'] = parse_timestamp_to_datestring(create_time_epoch) incident_name = result.get('Name') or 'New {} from arcsight at {}'.format(type_of_incident, datetime.now()) labels = [{'type': key.encode('utf-8'), 'value': value.encode('utf-8') if value else value} for key, value in result.items()] incident = { 'name': incident_name, 'occurred': result['Create Time'], 'labels': labels, 'rawJSON': json.dumps(result) } incidents.append(incident) if len(incidents) >= FETCH_CHUNK_SIZE: break if len(already_fetched) > MAX_UNIQUE: already_fetched.pop(0) already_fetched.append(r_id) last_run = { 'already_fetched': already_fetched, } demisto.setLastRun({'value': json.dumps(last_run)}) decode_arcsight_output(incidents) if demisto.command() == 'as-fetch-incidents': contents = { 'last_run': last_run, 'last_run_updated': demisto.getLastRun(), 'incidents': incidents, 'already_fetched': already_fetched } return_outputs(readable_output='', outputs={}, raw_response=contents) else: demisto.incidents(incidents)
def fetch_incidents(): last_fetched_event_timestamp = demisto.getLastRun().get('last_fetched_event_timestamp') if last_fetched_event_timestamp is not None: last_fetched_event_timestamp = datetime.strptime(last_fetched_event_timestamp, '%Y-%m-%dT%H:%M:%S.%f') else: last_fetched_event_timestamp, _ = parse_date_range(FIRST_FETCH_TIMESTAMP) # Need sometime in the future, so the timestamp will be taken from the query service_end_date_epoch = int(datetime.now().strftime('%s')) + 1000 global FETCH_QUERY if 'panw.' in FETCH_QUERY: service_start_date_epoch = int(last_fetched_event_timestamp.strftime('%s')) FETCH_QUERY += f' WHERE receive_time>{service_start_date_epoch}' elif 'tms.' in FETCH_QUERY: service_start_date_iso = last_fetched_event_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ') FETCH_QUERY += f" WHERE serverTime>'{service_start_date_iso}'" query_data = { 'query': FETCH_QUERY, 'startTime': 0, 'endTime': service_end_date_epoch, } response = query_loggings(query_data) try: result = response.json()['result'] pages = result['esResult']['hits']['hits'] except ValueError: raise Exception('Failed to parse the response from Cortex') incident_pairs = [] max_fetched_event_timestamp = last_fetched_event_timestamp for page in pages: incident, time_received = convert_log_to_incident(page) if 'panw.' in FETCH_QUERY: time_received_dt = datetime.fromtimestamp(time_received) elif 'tms.' in FETCH_QUERY: time_received_dt = datetime.strptime(time_received, '%Y-%m-%dT%H:%M:%S.%fZ') incident_pairs.append((incident, time_received_dt)) if incident_pairs: incidents, max_fetched_event_timestamp = process_incident_pairs(incident_pairs, 100) # max 100 per run demisto.setLastRun({ 'last_fetched_event_timestamp': max_fetched_event_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f') }) demisto.incidents(incidents) else: demisto.incidents([])
def fetch_incidents(): params = demisto.params() user_key = params.get('queryUserKey') user_key = user_key if user_key else ADMIN_EMAIL query = '' if params['query'] is None else params['query'] last_run = demisto.getLastRun() last_fetch = last_run.get('time') # handle first time fetch if last_fetch is None: last_fetch = datetime.datetime.now() - datetime.timedelta(days=1) else: last_fetch = datetime.datetime.strptime( last_fetch, '%Y-%m-%dT%H:%M:%SZ') current_fetch = last_fetch credentials = get_credentials( ['https://www.googleapis.com/auth/gmail.readonly', ], user_key) service = discovery.build('gmail', 'v1', credentials=credentials) query += last_fetch.strftime(' after:%Y/%m/%d') LOG('GMAIL: fetch parameters:\nuser: %s\nquery=%s\nfetch time: %s' % (user_key, query, last_fetch, )) result = service.users().messages().list( userId=user_key, maxResults=100, q=query).execute() incidents = [] # so far, so good LOG('GMAIL: possible new incidents are %s' % (result, )) for msg in result.get('messages', []): msg_result = service.users().messages().get( id=msg['id'], userId=user_key).execute() incident = mail_to_incident(msg_result, service, user_key) temp_date = datetime.datetime.strptime( incident['occurred'], '%Y-%m-%dT%H:%M:%SZ') # update last run if temp_date > last_fetch: last_fetch = temp_date + datetime.timedelta(seconds=1) # avoid duplication due to weak time query if temp_date > current_fetch: incidents.append(incident) demisto.info('extract {} incidents'.format(len(incidents))) demisto.setLastRun({'time': last_fetch.isoformat().split('.')[0] + 'Z'}) return incidents
def fetch_incidents(query, id_offset=None, fetch_by_created=None, **_): last_run = demisto.getLastRun() demisto.debug(f"last_run: {last_run}" if last_run else 'last_run is empty') id_offset = last_run.get("idOffset") if (last_run and last_run.get("idOffset")) else id_offset incidents, max_results = [], 50 if id_offset: query = f'{query} AND id >= {id_offset}' if fetch_by_created: query = f'{query} AND created>-1m' res = run_query(query, '', max_results) for ticket in res.get('issues'): id_offset = max(id_offset, ticket.get("id")) incidents.append(create_incident_from_ticket(ticket)) demisto.setLastRun({"idOffset": id_offset}) demisto.incidents(incidents)
def fetch_incidents_command(): objects_names_to_fetch = OBJECTS_TO_FETCH fetch_attachments = FETCH_ATTACHMENTS max_result = MAX_RESULT fetch_time = FETCH_TIME query_string = QUERY_STRING incidents = fetch_incidents(objects_names_to_fetch, fetch_time, max_result, query_string, fetch_attachments, real_fetch=True) if incidents: last_incident_created_time = incidents[-1].get('CreatedDateTime') next_created_time_to_fetch = \ (datetime.strptime(last_incident_created_time, DATE_FORMAT) + timedelta(seconds=1)).strftime(DATE_FORMAT) demisto.setLastRun({ 'last_created_time': next_created_time_to_fetch, 'objects_names_to_fetch': objects_names_to_fetch }) return
def fetch_incidents(): query_id = GetEventQuery() res = GetIncidentsByOrg(query_id) known_ids = demisto.getLastRun().get('ids', None) if known_ids is None or not known_ids: known_ids = [] incidents = [] for inc in res: if inc.get('incidentId') not in known_ids: incidents.append({"name": inc.get('eventName', 'New FortiSIEM Event'), "rawJSON": json.dumps(inc)}) if len(known_ids) >= 1000: known_ids.pop(0) known_ids.append(inc.get('incidentId')) demisto.setLastRun({ 'ids': known_ids, 'extended_keys': EXTENDED_KEYS }) demisto.incidents(incidents) sys.exit(0)
def fetch_incidents(): incidents = [] last_run = demisto.getLastRun() if last_run and last_run['time_stamp']: last_update_time = last_run['time_stamp'] else: # In first run last_update_time, _ = parse_date_range(FETCH_TIME, date_format='%Y-%m-%dT%H:%M:%S.%f'[:-3]) max_timestamp = last_update_time if FETCH_BEHAVIORS: params = 'behaviors.time_stamp>' + last_update_time behaviors = fetch_behaviors_request(params) for behavior in behaviors.get('results'): incident = behavior_to_incident(behavior) # 0 corresponds to never triggered time_stamp = behavior.get('behaviors.time_stamp')[:-5] # comapre time_stamp if time_stamp > max_timestamp: max_timestamp = time_stamp incidents.append(incident) if FETCH_NOTIFICATIONS: params = 'notifications.time_stamp>' + last_update_time notifications = search_notifications_request(params) for notification in notifications.get('results'): incident = notifications_to_incidents(notification) time_stamp = notification.get('notifications.time_stamp')[:-5] if time_stamp > max_timestamp: max_timestamp = time_stamp incidents.append(incident) demisto.setLastRun({ 'time_stamp': max_timestamp }) demisto.incidents(incidents)
def fetch_incidents(): """ Fetch events from this integration and return them as Demisto incidents returns: Demisto incidents """ # demisto.getLastRun() will returns an obj with the previous run in it. last_run = demisto.getLastRun() # Get the last fetch time and data if it exists last_fetch = last_run.get('last_fetched_data_timestamp') last_fetched_data = last_run.get('last_fetched_data') # Handle first time fetch, fetch incidents retroactively if not last_fetch: last_fetch, _ = parse_date_range(FETCH_TIME, to_timestamp=True) args = {'rows': MAX_ROWS, 'query': FETCH_QUERY} column_descriptions, data = snowflake_query(args) data.sort(key=lambda k: k[DATETIME_COLUMN]) # convert the data/events to demisto incidents incidents = [] for row in data: incident = row_to_incident(column_descriptions, row) incident_timestamp = incident.get('timestamp') # Update last run and add incident if the incident is newer than last fetch if incident_timestamp and incident_timestamp >= last_fetch: last_fetch = incident_timestamp if incident.get('rawJSON') != last_fetched_data: last_fetched_data = incident.get('rawJSON') del incident['timestamp'] incidents.append(incident) this_run = { 'last_fetched_data': last_fetched_data, 'last_fetched_data_timestamp': last_fetch } demisto.setLastRun(this_run) demisto.incidents(incidents)
def fetch_incidents_command(): """ Fetch alerts from Canary Tools as incidents in Demisto last_fetch: The latest fetched alert creation time """ last_fetch = demisto.getLastRun().get('time') if last_fetch is None: last_fetch = parse_date_range(FETCH_DELTA, '%Y-%m-%d-%H:%M:%S')[0] # All alerts retrieved from get_alerts are newer than last_fetch and are in a chronological order alerts = get_alerts(last_fetch) incidents = [] current_fetch = last_fetch for alert in alerts: current_fetch = 1000 * (int(demisto.get(alert, 'description.created'))) current_fetch = timestamp_to_datestring(current_fetch, '%Y-%m-%d-%H:%M:%S') incident = create_incident(alert) incidents.append(incident) demisto.incidents(incidents) demisto.setLastRun({'time': current_fetch})
def fetch_incidents(): """ Generates and fetches phishing email-like incidents Generates phishing email-like incidents, with the number of incidents, the speed of generation and the recurring time period all set by the values entered in the integration instance parameters. This method operates under the assumption that fetch-incidents is called once per minute. returns: Email-like incidents """ try: update_parameters() if not FREQUENCY: # Run once last_run = 0 if not demisto.getLastRun() else demisto.getLastRun().get('numOfIncidentsCreated', 0) num_of_incidents_created, incidents = generate_incidents(last_run) demisto.incidents(incidents) demisto.setLastRun({'numOfIncidentsCreated': last_run + num_of_incidents_created}) return else: minutes_of_generation = MAX_NUM_OF_INCIDENTS / float(INCIDENTS_PER_MINUTE) if minutes_of_generation > FREQUENCY: err_msg = 'The maximum number of incidents divided by the incidents to generate per minute' err_msg += ' exceeds every how often incidents should be generated. Please increase the value' err_msg += ' entered for how often the integration should generate incidents.' raise Exception(err_msg) run_counter = 0 if not demisto.getLastRun() else demisto.getLastRun().get('run_count', 0) last_run = 0 if not demisto.getLastRun() else demisto.getLastRun().get('numOfIncidentsCreated', 0) should_run = run_counter % FREQUENCY if should_run < math.ceil(minutes_of_generation): # then should run if should_run == 0: last_run = 0 num_of_incidents_created, incidents = generate_incidents(last_run) demisto.incidents(incidents) total_incidents_created = last_run + num_of_incidents_created updated_run_count = run_counter + 1 demisto.setLastRun({ 'numOfIncidentsCreated': total_incidents_created, 'run_count': updated_run_count }) return else: updated_run_count = run_counter + 1 demisto.setLastRun({ 'numOfIncidentsCreated': last_run, 'run_count': updated_run_count }) demisto.incidents([]) except Exception: raise
def fetch_incidents(): query_params = {} incidents = [] if FETCH_TIME: fetch_time = FETCH_TIME else: fetch_time = DEFAULTS['fetch_time'] last_run = demisto.getLastRun() if 'time' not in last_run: snow_time, _ = parse_date_range(fetch_time, '%Y-%m-%d %H:%M:%S') else: snow_time = last_run['time'] query = '' if SYSPARM_QUERY: query += SYSPARM_QUERY + '^' query += 'ORDERBY{0}^{0}>{1}'.format(TIMESTAMP_FIELD, snow_time) if query: query_params['sysparm_query'] = query query_params['sysparm_limit'] = SYSPARM_LIMIT path = 'table/' + TICKET_TYPE res = send_request(path, 'get', params=query_params) count = 0 parsed_snow_time = datetime.strptime(snow_time, '%Y-%m-%d %H:%M:%S') for result in res.get('result', []): labels = [] if TIMESTAMP_FIELD not in result: raise ValueError("The timestamp field [{}]" " does not exist in the ticket".format(TIMESTAMP_FIELD)) if count > SYSPARM_LIMIT: break try: if datetime.strptime(result[TIMESTAMP_FIELD], '%Y-%m-%d %H:%M:%S') < parsed_snow_time: continue except Exception: pass for k, v in result.iteritems(): if isinstance(v, basestring): labels.append({ 'type': k, 'value': v }) else: labels.append({ 'type': k, 'value': json.dumps(v) }) severity = SEVERITY_MAP.get(result.get('severity', ''), 0) file_names = [] if GET_ATTACHMENTS: file_entries = get_ticket_attachment_entries(result['sys_id']) for file_result in file_entries: if file_result['Type'] == entryTypes['error']: raise Exception('Error getting attachment: ' + str(file_result['Contents'])) file_names.append({ 'path': file_result['FileID'], 'name': file_result['File'] }) incidents.append({ 'name': 'ServiceNow Incident ' + result.get('number'), 'labels': labels, 'details': json.dumps(result), 'severity': severity, 'attachment': file_names, 'rawJSON': json.dumps(result) }) count += 1 snow_time = result[TIMESTAMP_FIELD] demisto.incidents(incidents) demisto.setLastRun({'time': snow_time})