def main(): params: Dict = demisto.params() base_url: str = params.get('base_url', '') client_id: str = params.get('client_id', '') client_secret: str = params.get('client_secret', '') event_type = ','.join(params.get('event_type', [])) verify_ssl = not params.get('insecure', False) proxy = params.get('proxy', False) offset = params.get('offset', '0') try: offset = int(offset) except ValueError: offset = 0 incident_type = params.get('incidentType', '') store_samples = params.get('store_samples', False) stream = EventStream(base_url=base_url, app_id='Demisto', verify_ssl=verify_ssl, proxy=proxy) LOG(f'Command being called is {demisto.command()}') try: if demisto.command() == 'test-module': run(test_module(base_url, client_id, client_secret, verify_ssl, proxy)) elif demisto.command() == 'long-running-execution': run(long_running_loop( base_url, client_id, client_secret, stream, offset, event_type, verify_ssl, proxy, incident_type, store_samples )) elif demisto.command() == 'crowdstrike-falcon-streaming-get-sample-events': get_sample_events(store_samples) except Exception as e: error_msg = f'Error in CrowdStrike Falcon Streaming v2: {str(e)}' demisto.error(error_msg) demisto.updateModuleHealth(error_msg) return_error(error_msg)
async def handle_listen_error(error: str): """ Logs an error and updates the module health accordingly. :param error: The error string. """ demisto.error(error) demisto.updateModuleHealth(error)
async def _discover_stream(self) -> None: """Discovers a CrowdStrike Falcon stream and initializes the data feed URL and refresh session URL client resource attributes Returns: None: No data returned. Raises: RuntimeError: In case stream discovery failed. """ await self.client.set_access_token(self.refresh_token) demisto.debug( f'Starting stream discovery. Container ID: {CONTAINER_ID}') discover_stream_response = await self.client.discover_stream( self.refresh_token) resources = discover_stream_response.get('resources', []) if not resources: demisto.updateModuleHealth( 'Did not discover event stream resources, verify the App ID is not used' ' in another integration instance') raise RuntimeError( f'Did not discover event stream resources - {str(discover_stream_response)}' ) resource = resources[0] self.data_feed_url = resource.get('dataFeedURL') demisto.debug(f'Discovered data feed URL: {self.data_feed_url}') self.session_token = resource.get('sessionToken', {}).get('token') refresh_url = resource.get('refreshActiveSessionURL') self.client.refresh_stream_url = refresh_url demisto.updateModuleHealth('') demisto.debug('Finished stream discovery successfully')
def handle_long_running_error(error: str): """ Handle errors in the long running process. Args: error: The error message. """ demisto.error(traceback.format_exc()) demisto.updateModuleHealth(error)
async def _http_request(self, method, url_suffix, full_url=None, headers=None, auth=None, json_data=None, # noqa: F841 params=None, data=None, files=None, timeout=10, resp_type='json', ok_codes=None, # noqa: F841 return_empty_response=False, retries=0, status_list_to_retry=None, # noqa: F841 backoff_factor=5, raise_on_redirect=False, raise_on_status=False, # noqa: F841 error_handler=None, **kwargs): # noqa: F841 while True: try: res = super()._http_request( method=method, url_suffix=url_suffix, params=params, full_url=full_url, resp_type='response', ok_codes=( OK_STATUS_CODE, CREATED_STATUS_CODE, UNAUTHORIZED_STATUS_CODE, TOO_MANY_REQUESTS_STATUS_CODE, NOT_FOUND_STATUS_CODE, ), ) if res.ok: try: demisto.debug(f'Got status code {res.status_code}') return res.json() except ValueError as e: demisto.debug( f'Failed deserializing the json-encoded content of the response: {res.content} - {str(e)}' ) elif res.status_code == UNAUTHORIZED_STATUS_CODE: sleep_time = uniform(1, 10) demisto.debug( f'Got status code 401 on stream discovery, going to sleep for {sleep_time} - {str(res.content)}' ) await sleep(sleep_time) demisto.debug('Getting new OAuth2 token') token = await kwargs.get('refresh_token').get_access_token() # type: ignore[union-attr] self.set_auth_headers(token) elif res.status_code == TOO_MANY_REQUESTS_STATUS_CODE: now_time = int(time.time()) retry_after = res.headers.get('X-Ratelimit-RetryAfter', 0) time_to_wait = max(int(retry_after), now_time + 5) - now_time demisto.debug(f'Rate limit exceeded, going to sleep for {time_to_wait} seconds and then retry. ' f'Response headers: {str(res.headers)} ' f'Response body: {str(res.content)}') demisto.updateModuleHealth( f'Rate limit exceeded, going to sleep for {time_to_wait} and then retry.' ) time.sleep(time_to_wait) demisto.debug('Finished waiting - retrying') elif res.status_code == NOT_FOUND_STATUS_CODE: demisto.debug(f'Got status code 404 - {str(res.content)}') return {} except Exception as e: demisto.debug(f'Got unexpected exception in the API HTTP request - {str(e)}')
def run_long_running(params: Dict, is_test: bool = False): """ Start the long running server :param params: Demisto params :param is_test: Indicates whether it's test-module run or regular run :return: None """ nginx_process = None nginx_log_monitor = None try: nginx_port = get_params_port(params) server_port = nginx_port + 1 # set our own log handlers APP.logger.removeHandler(default_handler) # pylint: disable=no-member integration_logger = IntegrationLogger() integration_logger.buffering = False log_handler = DemistoHandler(integration_logger) log_handler.setFormatter( logging.Formatter("flask log: [%(asctime)s] %(levelname)s in %(module)s: %(message)s") ) APP.logger.addHandler(log_handler) # pylint: disable=no-member demisto.debug('done setting demisto handler for logging') server = WSGIServer(('0.0.0.0', server_port), APP, log=DEMISTO_LOGGER, error_log=ERROR_LOGGER) if is_test: test_nginx_server(nginx_port, params) server_process = Process(target=server.serve_forever) server_process.start() time.sleep(2) try: server_process.terminate() server_process.join(1.0) except Exception as ex: demisto.error(f'failed stoping test wsgi server process: {ex}') else: nginx_process = start_nginx_server(nginx_port, params) nginx_log_monitor = gevent.spawn(nginx_log_monitor_loop, nginx_process) demisto.updateModuleHealth('') server.serve_forever() except Exception as e: error_message = str(e) demisto.error(f'An error occurred: {error_message}. Exception: {traceback.format_exc()}') demisto.updateModuleHealth(f'An error occurred: {error_message}') raise ValueError(error_message) finally: if nginx_process: try: nginx_process.terminate() except Exception as ex: demisto.error(f'Failed stopping nginx process when exiting: {ex}') if nginx_log_monitor: try: nginx_log_monitor.kill(timeout=1.0) except Exception as ex: demisto.error(f'Failed stopping nginx_log_monitor when exiting: {ex}')
def run_server(taxii_server: TAXIIServer, is_test=False): """ Start the taxii server. """ certificate_path = str() private_key_path = str() ssl_args = dict() try: if taxii_server.certificate and taxii_server.private_key and not taxii_server.http_server: certificate_file = NamedTemporaryFile(delete=False) certificate_path = certificate_file.name certificate_file.write(bytes(taxii_server.certificate, 'utf-8')) certificate_file.close() private_key_file = NamedTemporaryFile(delete=False) private_key_path = private_key_file.name private_key_file.write(bytes(taxii_server.private_key, 'utf-8')) private_key_file.close() context = SSLContext(PROTOCOL_TLSv1_2) context.load_cert_chain(certificate_path, private_key_path) ssl_args['ssl_context'] = context demisto.debug('Starting HTTPS Server') else: demisto.debug('Starting HTTP Server') wsgi_server = WSGIServer(('0.0.0.0', taxii_server.port), APP, **ssl_args, log=DEMISTO_LOGGER) if is_test: server_process = Process(target=wsgi_server.serve_forever) server_process.start() time.sleep(5) server_process.terminate() else: demisto.updateModuleHealth('') wsgi_server.serve_forever() except SSLError as e: ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}' handle_long_running_error(ssl_err_message) raise ValueError(ssl_err_message) except Exception as e: handle_long_running_error(f'An error occurred: {str(e)}') raise ValueError(str(e)) finally: if certificate_path: os.unlink(certificate_path) if private_key_path: os.unlink(private_key_path)
def long_running_loop(): """ Runs in a long running container - checking for newly mirrored investigations. """ while True: try: check_for_mirrors() except Exception as e: error = 'An error occurred: {}'.format(str(e)) demisto.error(error) demisto.updateModuleHealth(error) finally: time.sleep(5)
def main(): params: Dict = demisto.params() base_url: str = params.get('base_url', '') client_id: str = params.get('client_id', '') client_secret: str = params.get('client_secret', '') event_type = ','.join(params.get('event_type', []) or []) verify_ssl = not params.get('insecure', False) proxy = params.get('proxy', False) offset = params.get('offset', '0') try: offset = int(offset) except ValueError: offset = 0 incident_type = params.get('incidentType', '') store_samples = params.get('store_samples', False) first_fetch_time, _ = parse_date_range(params.get('fetch_time', '1 hour')) app_id = params.get('app_id') or 'Demisto' if not re.match(r'^[A-Za-z0-9]{0,32}$', app_id): raise ValueError( 'App ID is invalid: Must be a max. of 32 alphanumeric characters (a-z, A-Z, 0-9).' ) sock_read = int(params.get('sock_read_timeout', 120)) stream = EventStream(base_url=base_url, app_id=app_id, verify_ssl=verify_ssl, proxy=proxy) LOG(f'Command being called is {demisto.command()}') try: merge_integration_context() if demisto.command() == 'test-module': run( test_module(base_url, client_id, client_secret, verify_ssl, proxy)) elif demisto.command() == 'long-running-execution': run( long_running_loop(base_url, client_id, client_secret, stream, offset, event_type, verify_ssl, proxy, incident_type, first_fetch_time, store_samples, sock_read)) elif demisto.command() == 'fetch-incidents': fetch_samples() elif demisto.command( ) == 'crowdstrike-falcon-streaming-get-sample-events': get_sample_events(store_samples) except Exception as e: error_msg = f'Error in CrowdStrike Falcon Streaming v2: {str(e)}' demisto.error(error_msg) demisto.updateModuleHealth(error_msg) return_error(error_msg)
async def _refresh_stream(self) -> None: """Refreshes a CrowdStrike Falcon stream resource Returns: None: No data returned. Raises: RuntimeError: In case stream refresh failed. """ demisto.debug(f'Starting stream refresh. Container ID: {CONTAINER_ID}') response = await self.client.refresh_stream_session(self.refresh_token) if not response: # Should get here in case we got unexpected status code (e.g. 404) from the refresh stream query demisto.updateModuleHealth( 'Failed refreshing stream session, will try to discover new stream.' ) raise RuntimeError( 'Failed refreshing stream session. ' 'More details about the failure reason should appear in the logs above.' ) else: demisto.debug(f'Refresh stream response: {response}') demisto.updateModuleHealth('') demisto.debug('Finished stream refresh successfully')
async def _discover_refresh_stream(self, event: Event) -> None: """Discovers or refreshes a discovered CrowdStrike Falcon stream in a loop. Sleeps for 25 minutes (expiry time is 30 minutes) between operations. Args: event (Event): Asynchronous event object to set or clear its internal flag. Yields: Iterator[Dict]: Event fetched from the stream. """ client = Client(base_url=self.base_url, app_id=self.app_id, verify_ssl=self.verify_ssl, proxy=self.proxy) while True: refreshed = True if client.refresh_stream_url: # We already discovered an event stream, need to refresh it demisto.debug('Starting stream refresh') try: response = await client.refresh_stream_session(self.refresh_token) demisto.debug(f'Refresh stream response: {response}') except Exception as e: demisto.updateModuleHealth('Failed refreshing stream session, will retry in 30 seconds.') demisto.debug(f'Failed refreshing stream session: {e}') refreshed = False else: demisto.updateModuleHealth('') demisto.debug('Finished stream refresh successfully') else: # We have no event stream, need to discover await client.set_access_token(self.refresh_token) demisto.debug('Starting stream discovery') discover_stream_response = await client.discover_stream(self.refresh_token) demisto.debug('Finished stream discovery') resources = discover_stream_response.get('resources', []) if not resources: demisto.updateModuleHealth('Did not discover event stream resources, verify the App ID is not used' ' in another integration instance') demisto.error(f'Did not discover event stream resources - {str(discover_stream_response)}') return resource = resources[0] self.data_feed_url = resource.get('dataFeedURL') demisto.debug(f'Discovered data feed URL: {self.data_feed_url}') self.session_token = resource.get('sessionToken', {}).get('token') refresh_url = resource.get('refreshActiveSessionURL') client.refresh_stream_url = refresh_url event.set() if refreshed: demisto.debug('Discover/Refresh loop - going to sleep for 25 minutes') await sleep(MINUTES_25) event.clear() else: demisto.debug('Failed refreshing stream, going to sleep for 30 seconds and then retry') await sleep(30)
async def listen(**payload): """ Listens to Slack RTM messages :param payload: The message payload """ data: dict = payload.get('data', {}) data_type: str = payload.get('type', '') client: slack.WebClient = payload.get('web_client') if data_type == 'error': error = payload.get('error', {}) await handle_listen_error( 'Slack API has thrown an error. Code: {}, Message: {}.'.format( error.get('code'), error.get('msg'))) return try: subtype = data.get('subtype', '') text = data.get('text', '') user_id = data.get('user', '') channel = data.get('channel', '') message_bot_id = data.get('bot_id', '') if subtype == 'bot_message' or message_bot_id: return integration_context = demisto.getIntegrationContext() user = await get_user_by_id_async(client, integration_context, user_id) if channel and channel[0] == 'D': # DM await handle_dm(user, text, client) elif await check_and_handle_entitlement(text, user): await client.chat_postMessage( channel=channel, text='Response received by {}'.format(user.get('name'))) else: if not integration_context or 'mirrors' not in integration_context: return channel_id = data.get('channel') mirrors = json.loads(integration_context['mirrors']) mirror_filter = list( filter(lambda m: m['channel_id'] == channel_id, mirrors)) if not mirror_filter: return for mirror in mirror_filter: if mirror['mirror_direction'] == 'FromDemisto' or mirror[ 'mirror_type'] == 'none': return if not mirror['mirrored']: # In case the investigation is not mirrored yet mirror = mirrors.pop(mirrors.index(mirror)) if mirror['mirror_to'] and mirror[ 'mirror_direction'] and mirror['mirror_type']: investigation_id = mirror['investigation_id'] mirror_type = mirror['mirror_type'] auto_close = mirror['auto_close'] direction = mirror['mirror_direction'] if isinstance(auto_close, str): auto_close = bool(strtobool(auto_close)) demisto.info( 'Mirroring: {}'.format(investigation_id)) demisto.mirrorInvestigation( investigation_id, '{}:{}'.format(mirror_type, direction), auto_close) mirror['mirrored'] = True mirrors.append(mirror) set_to_latest_integration_context('mirrors', mirrors) investigation_id = mirror['investigation_id'] await handle_text(client, investigation_id, text, user) # Reset module health demisto.updateModuleHealth("") except Exception as e: await handle_listen_error( 'Error occurred while listening to Slack: {}'.format(str(e)))
async def fetch_event( self, first_fetch_time: datetime, initial_offset: int = 0, event_type: str = '', sock_read: int = 120 ) -> AsyncGenerator[Dict, None]: """Retrieves events from a CrowdStrike Falcon stream starting from given offset. Args: first_fetch_time (datetime): The start time to fetch from retroactively for the first fetch. initial_offset (int): Stream offset to start the fetch from. event_type (str): Stream event type to fetch. sock_read (int) Client session sock read timeout. Yields: AsyncGenerator[Dict, None]: Event fetched from the stream. """ while True: demisto.debug('Fetching event') try: event = Event() create_task(self._discover_refresh_stream(event)) demisto.debug('Waiting for stream discovery with 10 seconds timeout') await wait_for(event.wait(), 10) except TimeoutError as e: demisto.debug(f'Failed discovering stream: {e} - ' f'Going to sleep for 30 seconds and then retry - {traceback.format_exc()}') await sleep(30) else: demisto.debug('Done waiting for stream discovery') events_fetched = 0 new_lines_fetched = 0 last_fetch_stats_print = datetime.utcnow() async with ClientSession( connector=TCPConnector(ssl=self.verify_ssl), headers={ 'Authorization': f'Token {self.session_token}', 'Connection': 'keep-alive' }, trust_env=self.proxy, timeout=ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=sock_read) ) as session: try: integration_context = get_integration_context() offset = integration_context.get('offset', 0) or initial_offset demisto.debug(f'Starting to fetch from offset {offset} events of type {event_type} ' f'from time {first_fetch_time}') async with session.get( self.data_feed_url, params={'offset': offset, 'eventType': event_type}, timeout=ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=sock_read) ) as res: demisto.updateModuleHealth('') demisto.debug(f'Fetched event: {res.content}') async for line in res.content: stripped_line = line.strip() if stripped_line: events_fetched += 1 try: streaming_event = json.loads(stripped_line) event_metadata = streaming_event.get('metadata', {}) event_creation_time = event_metadata.get('eventCreationTime', 0) if not event_creation_time: demisto.debug( 'Could not extract "eventCreationTime" field, using 0 instead. ' f'{streaming_event}') else: event_creation_time /= 1000 event_creation_time_dt = datetime.fromtimestamp(event_creation_time) if event_creation_time_dt < first_fetch_time: demisto.debug(f'Event with offset {event_metadata.get("offset")} ' f'and creation time {event_creation_time} was skipped.') continue yield streaming_event except json.decoder.JSONDecodeError: demisto.debug(f'Failed decoding event (skipping it) - {str(stripped_line)}') else: new_lines_fetched += 1 if last_fetch_stats_print + timedelta(minutes=1) <= datetime.utcnow(): demisto.info( f'Fetched {events_fetched} events and' f' {new_lines_fetched} new lines' f' from the stream in the last minute.') events_fetched = 0 new_lines_fetched = 0 last_fetch_stats_print = datetime.utcnow() except Exception as e: demisto.debug(f'Failed to fetch event: {e} - Going to sleep for 10 seconds and then retry -' f' {traceback.format_exc()}') await sleep(10)
def main() -> None: demisto.debug(f'Command being called is {demisto.command()}') try: try: port = int(demisto.params().get('longRunningPort')) except ValueError as e: raise ValueError(f'Invalid listen port - {e}') if demisto.command() == 'test-module': return_results('ok') elif demisto.command() == 'fetch-incidents': fetch_samples() elif demisto.command() == 'long-running-execution': while True: certificate = demisto.params().get('certificate', '') private_key = demisto.params().get('key', '') certificate_path = '' private_key_path = '' try: ssl_args = dict() if certificate and private_key: certificate_file = NamedTemporaryFile(delete=False) certificate_path = certificate_file.name certificate_file.write(bytes(certificate, 'utf-8')) certificate_file.close() ssl_args['ssl_certfile'] = certificate_path private_key_file = NamedTemporaryFile(delete=False) private_key_path = private_key_file.name private_key_file.write(bytes(private_key, 'utf-8')) private_key_file.close() ssl_args['ssl_keyfile'] = private_key_path demisto.debug('Starting HTTPS Server') else: demisto.debug('Starting HTTP Server') integration_logger = IntegrationLogger() integration_logger.buffering = False log_config = dict(uvicorn.config.LOGGING_CONFIG) log_config['handlers']['default'][ 'stream'] = integration_logger log_config['handlers']['access'][ 'stream'] = integration_logger log_config['formatters']['access'] = { '()': GenericWebhookAccessFormatter, 'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s "%(user_agent)s"' } uvicorn.run(app, host='0.0.0.0', port=port, log_config=log_config, **ssl_args) except Exception as e: demisto.error( f'An error occurred in the long running loop: {str(e)} - {format_exc()}' ) demisto.updateModuleHealth(f'An error occurred: {str(e)}') finally: if certificate_path: os.unlink(certificate_path) if private_key_path: os.unlink(private_key_path) time.sleep(5) except Exception as e: demisto.error(format_exc()) return_error( f'Failed to execute {demisto.command()} command. Error: {e}')
def run_long_running(params, is_test=False): """ Start the long running server :param params: Demisto params :param is_test: Indicates whether it's test-module run or regular run :return: None """ certificate: str = params.get('certificate', '') private_key: str = params.get('key', '') certificate_path = str() private_key_path = str() try: port = get_params_port(params) ssl_args = dict() if (certificate and not private_key) or (private_key and not certificate): raise DemistoException( 'If using HTTPS connection, both certificate and private key should be provided.' ) if certificate and private_key: certificate_file = NamedTemporaryFile(delete=False) certificate_path = certificate_file.name certificate_file.write(bytes(certificate, 'utf-8')) certificate_file.close() private_key_file = NamedTemporaryFile(delete=False) private_key_path = private_key_file.name private_key_file.write(bytes(private_key, 'utf-8')) private_key_file.close() context = SSLContext(PROTOCOL_TLSv1_2) context.load_cert_chain(certificate_path, private_key_path) ssl_args['ssl_context'] = context demisto.debug('Starting HTTPS Server') else: demisto.debug('Starting HTTP Server') server = WSGIServer(('0.0.0.0', port), APP, **ssl_args, log=DEMISTO_LOGGER) if is_test: server_process = Process(target=server.serve_forever) server_process.start() time.sleep(5) server_process.terminate() else: demisto.updateModuleHealth('') server.serve_forever() except SSLError as e: ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}' demisto.error(ssl_err_message) demisto.updateModuleHealth(f'An error occurred: {ssl_err_message}') raise ValueError(ssl_err_message) except Exception as e: error_message = str(e) demisto.error( f'An error occurred in long running loop: {error_message}') demisto.updateModuleHealth(f'An error occurred: {error_message}') raise ValueError(error_message) finally: if certificate_path: os.unlink(certificate_path) if private_key_path: os.unlink(private_key_path)