def main():
params: Dict = demisto.params()
base_url: str = params.get('base_url', '')
client_id: str = params.get('client_id', '')
client_secret: str = params.get('client_secret', '')
event_type = ','.join(params.get('event_type', []))
verify_ssl = not params.get('insecure', False)
proxy = params.get('proxy', False)
offset = params.get('offset', '0')
try:
offset = int(offset)
except ValueError:
offset = 0
incident_type = params.get('incidentType', '')
store_samples = params.get('store_samples', False)
stream = EventStream(base_url=base_url, app_id='Demisto', verify_ssl=verify_ssl, proxy=proxy)
LOG(f'Command being called is {demisto.command()}')
try:
if demisto.command() == 'test-module':
run(test_module(base_url, client_id, client_secret, verify_ssl, proxy))
elif demisto.command() == 'long-running-execution':
run(long_running_loop(
base_url, client_id, client_secret, stream, offset, event_type, verify_ssl, proxy, incident_type,
store_samples
))
elif demisto.command() == 'crowdstrike-falcon-streaming-get-sample-events':
get_sample_events(store_samples)
except Exception as e:
error_msg = f'Error in CrowdStrike Falcon Streaming v2: {str(e)}'
demisto.error(error_msg)
demisto.updateModuleHealth(error_msg)
return_error(error_msg)
async def handle_listen_error(error: str):
"""
Logs an error and updates the module health accordingly.
:param error: The error string.
"""
demisto.error(error)
demisto.updateModuleHealth(error)
async def _discover_stream(self) -> None:
"""Discovers a CrowdStrike Falcon stream
and initializes the data feed URL and refresh session URL client resource attributes
Returns:
None: No data returned.
Raises:
RuntimeError: In case stream discovery failed.
"""
await self.client.set_access_token(self.refresh_token)
demisto.debug(
f'Starting stream discovery. Container ID: {CONTAINER_ID}')
discover_stream_response = await self.client.discover_stream(
self.refresh_token)
resources = discover_stream_response.get('resources', [])
if not resources:
demisto.updateModuleHealth(
'Did not discover event stream resources, verify the App ID is not used'
' in another integration instance')
raise RuntimeError(
f'Did not discover event stream resources - {str(discover_stream_response)}'
)
resource = resources[0]
self.data_feed_url = resource.get('dataFeedURL')
demisto.debug(f'Discovered data feed URL: {self.data_feed_url}')
self.session_token = resource.get('sessionToken', {}).get('token')
refresh_url = resource.get('refreshActiveSessionURL')
self.client.refresh_stream_url = refresh_url
demisto.updateModuleHealth('')
demisto.debug('Finished stream discovery successfully')
def handle_long_running_error(error: str):
"""
Handle errors in the long running process.
Args:
error: The error message.
"""
demisto.error(traceback.format_exc())
demisto.updateModuleHealth(error)
async def _http_request(self, method, url_suffix, full_url=None, headers=None, auth=None, json_data=None, # noqa: F841
params=None, data=None, files=None, timeout=10, resp_type='json', ok_codes=None, # noqa: F841
return_empty_response=False, retries=0, status_list_to_retry=None, # noqa: F841
backoff_factor=5, raise_on_redirect=False, raise_on_status=False, # noqa: F841
error_handler=None, **kwargs): # noqa: F841
while True:
try:
res = super()._http_request(
method=method,
url_suffix=url_suffix,
params=params,
full_url=full_url,
resp_type='response',
ok_codes=(
OK_STATUS_CODE,
CREATED_STATUS_CODE,
UNAUTHORIZED_STATUS_CODE,
TOO_MANY_REQUESTS_STATUS_CODE,
NOT_FOUND_STATUS_CODE,
),
)
if res.ok:
try:
demisto.debug(f'Got status code {res.status_code}')
return res.json()
except ValueError as e:
demisto.debug(
f'Failed deserializing the json-encoded content of the response: {res.content} - {str(e)}'
)
elif res.status_code == UNAUTHORIZED_STATUS_CODE:
sleep_time = uniform(1, 10)
demisto.debug(
f'Got status code 401 on stream discovery, going to sleep for {sleep_time} - {str(res.content)}'
)
await sleep(sleep_time)
demisto.debug('Getting new OAuth2 token')
token = await kwargs.get('refresh_token').get_access_token() # type: ignore[union-attr]
self.set_auth_headers(token)
elif res.status_code == TOO_MANY_REQUESTS_STATUS_CODE:
now_time = int(time.time())
retry_after = res.headers.get('X-Ratelimit-RetryAfter', 0)
time_to_wait = max(int(retry_after), now_time + 5) - now_time
demisto.debug(f'Rate limit exceeded, going to sleep for {time_to_wait} seconds and then retry. '
f'Response headers: {str(res.headers)} '
f'Response body: {str(res.content)}')
demisto.updateModuleHealth(
f'Rate limit exceeded, going to sleep for {time_to_wait} and then retry.'
)
time.sleep(time_to_wait)
demisto.debug('Finished waiting - retrying')
elif res.status_code == NOT_FOUND_STATUS_CODE:
demisto.debug(f'Got status code 404 - {str(res.content)}')
return {}
except Exception as e:
demisto.debug(f'Got unexpected exception in the API HTTP request - {str(e)}')
def run_long_running(params: Dict, is_test: bool = False):
"""
Start the long running server
:param params: Demisto params
:param is_test: Indicates whether it's test-module run or regular run
:return: None
"""
nginx_process = None
nginx_log_monitor = None
try:
nginx_port = get_params_port(params)
server_port = nginx_port + 1
# set our own log handlers
APP.logger.removeHandler(default_handler) # pylint: disable=no-member
integration_logger = IntegrationLogger()
integration_logger.buffering = False
log_handler = DemistoHandler(integration_logger)
log_handler.setFormatter(
logging.Formatter("flask log: [%(asctime)s] %(levelname)s in %(module)s: %(message)s")
)
APP.logger.addHandler(log_handler) # pylint: disable=no-member
demisto.debug('done setting demisto handler for logging')
server = WSGIServer(('0.0.0.0', server_port), APP, log=DEMISTO_LOGGER, error_log=ERROR_LOGGER)
if is_test:
test_nginx_server(nginx_port, params)
server_process = Process(target=server.serve_forever)
server_process.start()
time.sleep(2)
try:
server_process.terminate()
server_process.join(1.0)
except Exception as ex:
demisto.error(f'failed stoping test wsgi server process: {ex}')
else:
nginx_process = start_nginx_server(nginx_port, params)
nginx_log_monitor = gevent.spawn(nginx_log_monitor_loop, nginx_process)
demisto.updateModuleHealth('')
server.serve_forever()
except Exception as e:
error_message = str(e)
demisto.error(f'An error occurred: {error_message}. Exception: {traceback.format_exc()}')
demisto.updateModuleHealth(f'An error occurred: {error_message}')
raise ValueError(error_message)
finally:
if nginx_process:
try:
nginx_process.terminate()
except Exception as ex:
demisto.error(f'Failed stopping nginx process when exiting: {ex}')
if nginx_log_monitor:
try:
nginx_log_monitor.kill(timeout=1.0)
except Exception as ex:
demisto.error(f'Failed stopping nginx_log_monitor when exiting: {ex}')
def run_server(taxii_server: TAXIIServer, is_test=False):
"""
Start the taxii server.
"""
certificate_path = str()
private_key_path = str()
ssl_args = dict()
try:
if taxii_server.certificate and taxii_server.private_key and not taxii_server.http_server:
certificate_file = NamedTemporaryFile(delete=False)
certificate_path = certificate_file.name
certificate_file.write(bytes(taxii_server.certificate, 'utf-8'))
certificate_file.close()
private_key_file = NamedTemporaryFile(delete=False)
private_key_path = private_key_file.name
private_key_file.write(bytes(taxii_server.private_key, 'utf-8'))
private_key_file.close()
context = SSLContext(PROTOCOL_TLSv1_2)
context.load_cert_chain(certificate_path, private_key_path)
ssl_args['ssl_context'] = context
demisto.debug('Starting HTTPS Server')
else:
demisto.debug('Starting HTTP Server')
wsgi_server = WSGIServer(('0.0.0.0', taxii_server.port),
APP,
**ssl_args,
log=DEMISTO_LOGGER)
if is_test:
server_process = Process(target=wsgi_server.serve_forever)
server_process.start()
time.sleep(5)
server_process.terminate()
else:
demisto.updateModuleHealth('')
wsgi_server.serve_forever()
except SSLError as e:
ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}'
handle_long_running_error(ssl_err_message)
raise ValueError(ssl_err_message)
except Exception as e:
handle_long_running_error(f'An error occurred: {str(e)}')
raise ValueError(str(e))
finally:
if certificate_path:
os.unlink(certificate_path)
if private_key_path:
os.unlink(private_key_path)
def long_running_loop():
"""
Runs in a long running container - checking for newly mirrored investigations.
"""
while True:
try:
check_for_mirrors()
except Exception as e:
error = 'An error occurred: {}'.format(str(e))
demisto.error(error)
demisto.updateModuleHealth(error)
finally:
time.sleep(5)
def main():
params: Dict = demisto.params()
base_url: str = params.get('base_url', '')
client_id: str = params.get('client_id', '')
client_secret: str = params.get('client_secret', '')
event_type = ','.join(params.get('event_type', []) or [])
verify_ssl = not params.get('insecure', False)
proxy = params.get('proxy', False)
offset = params.get('offset', '0')
try:
offset = int(offset)
except ValueError:
offset = 0
incident_type = params.get('incidentType', '')
store_samples = params.get('store_samples', False)
first_fetch_time, _ = parse_date_range(params.get('fetch_time', '1 hour'))
app_id = params.get('app_id') or 'Demisto'
if not re.match(r'^[A-Za-z0-9]{0,32}$', app_id):
raise ValueError(
'App ID is invalid: Must be a max. of 32 alphanumeric characters (a-z, A-Z, 0-9).'
)
sock_read = int(params.get('sock_read_timeout', 120))
stream = EventStream(base_url=base_url,
app_id=app_id,
verify_ssl=verify_ssl,
proxy=proxy)
LOG(f'Command being called is {demisto.command()}')
try:
merge_integration_context()
if demisto.command() == 'test-module':
run(
test_module(base_url, client_id, client_secret, verify_ssl,
proxy))
elif demisto.command() == 'long-running-execution':
run(
long_running_loop(base_url, client_id, client_secret, stream,
offset, event_type, verify_ssl, proxy,
incident_type, first_fetch_time,
store_samples, sock_read))
elif demisto.command() == 'fetch-incidents':
fetch_samples()
elif demisto.command(
) == 'crowdstrike-falcon-streaming-get-sample-events':
get_sample_events(store_samples)
except Exception as e:
error_msg = f'Error in CrowdStrike Falcon Streaming v2: {str(e)}'
demisto.error(error_msg)
demisto.updateModuleHealth(error_msg)
return_error(error_msg)
async def _refresh_stream(self) -> None:
"""Refreshes a CrowdStrike Falcon stream resource
Returns:
None: No data returned.
Raises:
RuntimeError: In case stream refresh failed.
"""
demisto.debug(f'Starting stream refresh. Container ID: {CONTAINER_ID}')
response = await self.client.refresh_stream_session(self.refresh_token)
if not response:
# Should get here in case we got unexpected status code (e.g. 404) from the refresh stream query
demisto.updateModuleHealth(
'Failed refreshing stream session, will try to discover new stream.'
)
raise RuntimeError(
'Failed refreshing stream session. '
'More details about the failure reason should appear in the logs above.'
)
else:
demisto.debug(f'Refresh stream response: {response}')
demisto.updateModuleHealth('')
demisto.debug('Finished stream refresh successfully')
async def _discover_refresh_stream(self, event: Event) -> None:
"""Discovers or refreshes a discovered CrowdStrike Falcon stream in a loop.
Sleeps for 25 minutes (expiry time is 30 minutes) between operations.
Args:
event (Event): Asynchronous event object to set or clear its internal flag.
Yields:
Iterator[Dict]: Event fetched from the stream.
"""
client = Client(base_url=self.base_url, app_id=self.app_id, verify_ssl=self.verify_ssl, proxy=self.proxy)
while True:
refreshed = True
if client.refresh_stream_url:
# We already discovered an event stream, need to refresh it
demisto.debug('Starting stream refresh')
try:
response = await client.refresh_stream_session(self.refresh_token)
demisto.debug(f'Refresh stream response: {response}')
except Exception as e:
demisto.updateModuleHealth('Failed refreshing stream session, will retry in 30 seconds.')
demisto.debug(f'Failed refreshing stream session: {e}')
refreshed = False
else:
demisto.updateModuleHealth('')
demisto.debug('Finished stream refresh successfully')
else:
# We have no event stream, need to discover
await client.set_access_token(self.refresh_token)
demisto.debug('Starting stream discovery')
discover_stream_response = await client.discover_stream(self.refresh_token)
demisto.debug('Finished stream discovery')
resources = discover_stream_response.get('resources', [])
if not resources:
demisto.updateModuleHealth('Did not discover event stream resources, verify the App ID is not used'
' in another integration instance')
demisto.error(f'Did not discover event stream resources - {str(discover_stream_response)}')
return
resource = resources[0]
self.data_feed_url = resource.get('dataFeedURL')
demisto.debug(f'Discovered data feed URL: {self.data_feed_url}')
self.session_token = resource.get('sessionToken', {}).get('token')
refresh_url = resource.get('refreshActiveSessionURL')
client.refresh_stream_url = refresh_url
event.set()
if refreshed:
demisto.debug('Discover/Refresh loop - going to sleep for 25 minutes')
await sleep(MINUTES_25)
event.clear()
else:
demisto.debug('Failed refreshing stream, going to sleep for 30 seconds and then retry')
await sleep(30)
async def listen(**payload):
"""
Listens to Slack RTM messages
:param payload: The message payload
"""
data: dict = payload.get('data', {})
data_type: str = payload.get('type', '')
client: slack.WebClient = payload.get('web_client')
if data_type == 'error':
error = payload.get('error', {})
await handle_listen_error(
'Slack API has thrown an error. Code: {}, Message: {}.'.format(
error.get('code'), error.get('msg')))
return
try:
subtype = data.get('subtype', '')
text = data.get('text', '')
user_id = data.get('user', '')
channel = data.get('channel', '')
message_bot_id = data.get('bot_id', '')
if subtype == 'bot_message' or message_bot_id:
return
integration_context = demisto.getIntegrationContext()
user = await get_user_by_id_async(client, integration_context, user_id)
if channel and channel[0] == 'D':
# DM
await handle_dm(user, text, client)
elif await check_and_handle_entitlement(text, user):
await client.chat_postMessage(
channel=channel,
text='Response received by {}'.format(user.get('name')))
else:
if not integration_context or 'mirrors' not in integration_context:
return
channel_id = data.get('channel')
mirrors = json.loads(integration_context['mirrors'])
mirror_filter = list(
filter(lambda m: m['channel_id'] == channel_id, mirrors))
if not mirror_filter:
return
for mirror in mirror_filter:
if mirror['mirror_direction'] == 'FromDemisto' or mirror[
'mirror_type'] == 'none':
return
if not mirror['mirrored']:
# In case the investigation is not mirrored yet
mirror = mirrors.pop(mirrors.index(mirror))
if mirror['mirror_to'] and mirror[
'mirror_direction'] and mirror['mirror_type']:
investigation_id = mirror['investigation_id']
mirror_type = mirror['mirror_type']
auto_close = mirror['auto_close']
direction = mirror['mirror_direction']
if isinstance(auto_close, str):
auto_close = bool(strtobool(auto_close))
demisto.info(
'Mirroring: {}'.format(investigation_id))
demisto.mirrorInvestigation(
investigation_id,
'{}:{}'.format(mirror_type, direction), auto_close)
mirror['mirrored'] = True
mirrors.append(mirror)
set_to_latest_integration_context('mirrors', mirrors)
investigation_id = mirror['investigation_id']
await handle_text(client, investigation_id, text, user)
# Reset module health
demisto.updateModuleHealth("")
except Exception as e:
await handle_listen_error(
'Error occurred while listening to Slack: {}'.format(str(e)))
async def fetch_event(
self, first_fetch_time: datetime, initial_offset: int = 0, event_type: str = '', sock_read: int = 120
) -> AsyncGenerator[Dict, None]:
"""Retrieves events from a CrowdStrike Falcon stream starting from given offset.
Args:
first_fetch_time (datetime): The start time to fetch from retroactively for the first fetch.
initial_offset (int): Stream offset to start the fetch from.
event_type (str): Stream event type to fetch.
sock_read (int) Client session sock read timeout.
Yields:
AsyncGenerator[Dict, None]: Event fetched from the stream.
"""
while True:
demisto.debug('Fetching event')
try:
event = Event()
create_task(self._discover_refresh_stream(event))
demisto.debug('Waiting for stream discovery with 10 seconds timeout')
await wait_for(event.wait(), 10)
except TimeoutError as e:
demisto.debug(f'Failed discovering stream: {e} - '
f'Going to sleep for 30 seconds and then retry - {traceback.format_exc()}')
await sleep(30)
else:
demisto.debug('Done waiting for stream discovery')
events_fetched = 0
new_lines_fetched = 0
last_fetch_stats_print = datetime.utcnow()
async with ClientSession(
connector=TCPConnector(ssl=self.verify_ssl),
headers={
'Authorization': f'Token {self.session_token}',
'Connection': 'keep-alive'
},
trust_env=self.proxy,
timeout=ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=sock_read)
) as session:
try:
integration_context = get_integration_context()
offset = integration_context.get('offset', 0) or initial_offset
demisto.debug(f'Starting to fetch from offset {offset} events of type {event_type} '
f'from time {first_fetch_time}')
async with session.get(
self.data_feed_url,
params={'offset': offset, 'eventType': event_type},
timeout=ClientTimeout(total=None, connect=60, sock_connect=60, sock_read=sock_read)
) as res:
demisto.updateModuleHealth('')
demisto.debug(f'Fetched event: {res.content}')
async for line in res.content:
stripped_line = line.strip()
if stripped_line:
events_fetched += 1
try:
streaming_event = json.loads(stripped_line)
event_metadata = streaming_event.get('metadata', {})
event_creation_time = event_metadata.get('eventCreationTime', 0)
if not event_creation_time:
demisto.debug(
'Could not extract "eventCreationTime" field, using 0 instead. '
f'{streaming_event}')
else:
event_creation_time /= 1000
event_creation_time_dt = datetime.fromtimestamp(event_creation_time)
if event_creation_time_dt < first_fetch_time:
demisto.debug(f'Event with offset {event_metadata.get("offset")} '
f'and creation time {event_creation_time} was skipped.')
continue
yield streaming_event
except json.decoder.JSONDecodeError:
demisto.debug(f'Failed decoding event (skipping it) - {str(stripped_line)}')
else:
new_lines_fetched += 1
if last_fetch_stats_print + timedelta(minutes=1) <= datetime.utcnow():
demisto.info(
f'Fetched {events_fetched} events and'
f' {new_lines_fetched} new lines'
f' from the stream in the last minute.')
events_fetched = 0
new_lines_fetched = 0
last_fetch_stats_print = datetime.utcnow()
except Exception as e:
demisto.debug(f'Failed to fetch event: {e} - Going to sleep for 10 seconds and then retry -'
f' {traceback.format_exc()}')
await sleep(10)
def main() -> None:
demisto.debug(f'Command being called is {demisto.command()}')
try:
try:
port = int(demisto.params().get('longRunningPort'))
except ValueError as e:
raise ValueError(f'Invalid listen port - {e}')
if demisto.command() == 'test-module':
return_results('ok')
elif demisto.command() == 'fetch-incidents':
fetch_samples()
elif demisto.command() == 'long-running-execution':
while True:
certificate = demisto.params().get('certificate', '')
private_key = demisto.params().get('key', '')
certificate_path = ''
private_key_path = ''
try:
ssl_args = dict()
if certificate and private_key:
certificate_file = NamedTemporaryFile(delete=False)
certificate_path = certificate_file.name
certificate_file.write(bytes(certificate, 'utf-8'))
certificate_file.close()
ssl_args['ssl_certfile'] = certificate_path
private_key_file = NamedTemporaryFile(delete=False)
private_key_path = private_key_file.name
private_key_file.write(bytes(private_key, 'utf-8'))
private_key_file.close()
ssl_args['ssl_keyfile'] = private_key_path
demisto.debug('Starting HTTPS Server')
else:
demisto.debug('Starting HTTP Server')
integration_logger = IntegrationLogger()
integration_logger.buffering = False
log_config = dict(uvicorn.config.LOGGING_CONFIG)
log_config['handlers']['default'][
'stream'] = integration_logger
log_config['handlers']['access'][
'stream'] = integration_logger
log_config['formatters']['access'] = {
'()':
GenericWebhookAccessFormatter,
'fmt':
'%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s "%(user_agent)s"'
}
uvicorn.run(app,
host='0.0.0.0',
port=port,
log_config=log_config,
**ssl_args)
except Exception as e:
demisto.error(
f'An error occurred in the long running loop: {str(e)} - {format_exc()}'
)
demisto.updateModuleHealth(f'An error occurred: {str(e)}')
finally:
if certificate_path:
os.unlink(certificate_path)
if private_key_path:
os.unlink(private_key_path)
time.sleep(5)
except Exception as e:
demisto.error(format_exc())
return_error(
f'Failed to execute {demisto.command()} command. Error: {e}')
def run_long_running(params, is_test=False):
"""
Start the long running server
:param params: Demisto params
:param is_test: Indicates whether it's test-module run or regular run
:return: None
"""
certificate: str = params.get('certificate', '')
private_key: str = params.get('key', '')
certificate_path = str()
private_key_path = str()
try:
port = get_params_port(params)
ssl_args = dict()
if (certificate and not private_key) or (private_key
and not certificate):
raise DemistoException(
'If using HTTPS connection, both certificate and private key should be provided.'
)
if certificate and private_key:
certificate_file = NamedTemporaryFile(delete=False)
certificate_path = certificate_file.name
certificate_file.write(bytes(certificate, 'utf-8'))
certificate_file.close()
private_key_file = NamedTemporaryFile(delete=False)
private_key_path = private_key_file.name
private_key_file.write(bytes(private_key, 'utf-8'))
private_key_file.close()
context = SSLContext(PROTOCOL_TLSv1_2)
context.load_cert_chain(certificate_path, private_key_path)
ssl_args['ssl_context'] = context
demisto.debug('Starting HTTPS Server')
else:
demisto.debug('Starting HTTP Server')
server = WSGIServer(('0.0.0.0', port),
APP,
**ssl_args,
log=DEMISTO_LOGGER)
if is_test:
server_process = Process(target=server.serve_forever)
server_process.start()
time.sleep(5)
server_process.terminate()
else:
demisto.updateModuleHealth('')
server.serve_forever()
except SSLError as e:
ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}'
demisto.error(ssl_err_message)
demisto.updateModuleHealth(f'An error occurred: {ssl_err_message}')
raise ValueError(ssl_err_message)
except Exception as e:
error_message = str(e)
demisto.error(
f'An error occurred in long running loop: {error_message}')
demisto.updateModuleHealth(f'An error occurred: {error_message}')
raise ValueError(error_message)
finally:
if certificate_path:
os.unlink(certificate_path)
if private_key_path:
os.unlink(private_key_path)
