def feed_main(feed_name, params=None, prefix=''):
if not params:
params = {k: v for k, v in demisto.params().items() if v is not None}
handle_proxy()
client = Client(**params)
command = demisto.command()
if command != 'fetch-indicators':
demisto.info('Command being called is {}'.format(command))
if prefix and not prefix.endswith('-'):
prefix += '-'
# Switch case
commands: dict = {
'test-module': module_test_command,
f'{prefix}get-indicators': get_indicators_command
}
try:
if command == 'fetch-indicators':
indicators = fetch_indicators_command(client,
params.get('indicator_type'))
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b) # type: ignore
else:
args = demisto.args()
args['feed_name'] = feed_name
readable_output, outputs, raw_response = commands[command](client,
args)
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
err_msg = f'Error in {feed_name} Integration - Encountered an issue with createIndicators' if \
'failed to create' in str(e) else f'Error in {feed_name} Integration [{e}]'
return_error(err_msg)
def main():
global EMAIL
command = demisto.command()
client = Client()
demisto.info(f'Command being called is {command}')
commands = {
'test-module': test_module,
'send-mail': send_mail_command,
'fetch-incidents': fetch_incidents,
'gmail-auth-link': auth_link_command,
'gmail-auth-test': auth_test_command,
}
try:
if command == 'fetch-incidents':
demisto.incidents(fetch_incidents(client))
sys.exit(0)
if command in commands:
demisto.results(commands[command](client))
# Log exceptions
except Exception as e:
import traceback
return_error('GMAIL: {}'.format(str(e)), traceback.format_exc())
def main():
max_indicators = get_limit(demisto.params().get('maxIndicators', MAX_INDICATORS), MAX_INDICATORS)
SESSION.proxies = handle_proxy()
client = SixgillFeedClient(demisto.params()['client_id'],
demisto.params()['client_secret'],
CHANNEL_CODE,
FeedStream.DARKFEED,
demisto, max_indicators, SESSION, VERIFY)
command = demisto.command()
demisto.info(f'Command being called is {command}')
tags = argToList(demisto.params().get('feedTags', []))
tlp_color = demisto.params().get('tlp_color')
commands: Dict[str, Callable] = {
'test-module': test_module_command,
'sixgill-get-indicators': get_indicators_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client, tags=tags, tlp_color=tlp_color)
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
demisto.error(traceback.format_exc())
return_error(f'Error failed to execute {demisto.command()}, error: [{e}]')
def fetch_incidents():
last_fetch = get_last_fetch_time()
es = elasticsearch_builder()
query = QueryString(query=FETCH_QUERY + " AND " + TIME_FIELD + ":*")
search = Search(using=es, index=FETCH_INDEX).filter(
{'range': {
TIME_FIELD: {
'gt': last_fetch
}
}})
search = search.sort({TIME_FIELD: {
'order': 'asc'
}})[0:FETCH_SIZE].query(query)
response = search.execute().to_dict()
_, total_results = get_total_results(response)
incidents = [] # type: List
if total_results > 0:
if 'Timestamp' in TIME_METHOD:
incidents, last_fetch = results_to_incidents_timestamp(
response, last_fetch)
demisto.setLastRun({'time': last_fetch})
else:
incidents, last_fetch = results_to_incidents_datetime(
response, last_fetch)
demisto.setLastRun(
{'time': datetime.strftime(last_fetch, TIME_FORMAT)})
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.incidents(incidents)
def generate_incidents(last_run):
"""
Determines how many incidents to create and generates them
parameter: (number) last_run
The number of incidents generated in the last fetch
returns:
The number of incidents generated in the current call to fetch_incidents and the incidents themselves
"""
if last_run > 0 and last_run > MAX_NUM_OF_INCIDENTS:
demisto.info('last_run is greater than MAX_NUM_OF_INCIDENTS')
return 0, []
incidents = []
num_of_incidents_left_to_create = MAX_NUM_OF_INCIDENTS - last_run
if num_of_incidents_left_to_create > INCIDENTS_PER_MINUTE:
num_of_incident_to_create = INCIDENTS_PER_MINUTE
else:
num_of_incident_to_create = num_of_incidents_left_to_create
for _ in range(num_of_incident_to_create):
email, email_object = create_email()
incidents.append({
'name': email_object.get('Subject'),
'details': email.as_string(),
'occurred': email_object.get('Date'),
'type': INCIDENT_TYPE,
'rawJSON': json.dumps(email_object)
})
return num_of_incident_to_create, incidents
def main():
params = demisto.params()
client = Client(params.get('api_key'), params.get('insecure'),
params.get('proxy'), params.get('indicator_feeds'),
params.get('custom_feed_urls'))
command = demisto.command()
demisto.info(f'Command being called is {command}')
# Switch case
commands = {
'test-module': module_test_command,
'autofocus-get-indicators': get_indicators_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
raise Exception(f'Error in {SOURCE_NAME} Integration [{e}]')
def update_alert_status():
"""
Updates the actionable alert status.
"""
args = demisto.args()
alert_status = args.get('alert_status')
alert_id = args.get('alert_id')
aggregate_alert_id = args.get('aggregate_alert_id')
demisto.info(
"update_alert_status: status - {}, alert_id - {}, aggregate_alert_id - {}"
.format(alert_status, alert_id, aggregate_alert_id))
aggregate_alert_id = [int(aggregate_alert_id)
] if aggregate_alert_id else aggregate_alert_id
alert_body = {"status": {"status": alert_status}}
sixgill_alerts_client = SixgillActionableAlertClient(
client_id=demisto.params()['client_id'],
client_secret=demisto.params()['client_secret'],
channel_id=CHANNEL_CODE,
logger=demisto,
session=SESSION,
verify=VERIFY)
res = sixgill_alerts_client.update_actionable_alert(
actionable_alert_id=alert_id,
json_body=alert_body,
sub_alert_indexes=aggregate_alert_id)
if res.get('status') == 200:
demisto.results("Actionable alert status updated")
def feed_main(feed_name, params, prefix=''):
params['feed_name'] = feed_name
client = Client(**params)
command = demisto.command()
if command != 'fetch-indicators':
demisto.info('Command being called is {}'.format(command))
if prefix and not prefix.endswith('-'):
prefix += '-'
# Switch case
commands: dict = {
'test-module': test_module,
f'{prefix}get-indicators': get_indicators_command
}
try:
if command == 'fetch-indicators':
indicators = fetch_indicators_command(client, params.get('indicator_type'))
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
args = demisto.args()
args['feed_name'] = feed_name
readable_output, outputs, raw_response = commands[command](client, args)
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
err_msg = f'Error in {feed_name} integration [{e}]\nTrace\n:{traceback.format_exc()}'
return_error(err_msg)
async def slack_loop():
"""
Starts a Slack RTM client while checking the connection.
"""
while True:
loop = asyncio.get_running_loop()
rtm_client = None
try:
rtm_client = slack.RTMClient(token=TOKEN,
run_async=True,
loop=loop,
auto_reconnect=False)
client_future = rtm_client.start()
while True:
await asyncio.sleep(10)
if rtm_client._websocket is None or rtm_client._websocket.closed or client_future.done(
):
ex = client_future.exception()
if ex:
demisto.error(
'Slack client raised an exception: {}'.format(ex))
demisto.info('Slack - websocket is closed or done')
break
except Exception as e:
error = 'Slack client raised an exception: {}'.format(e)
await handle_listen_error(error)
finally:
# If we got here, the websocket is closed or the client can't connect. Will try to connect every 5 seconds.
if rtm_client and not rtm_client._stopped:
rtm_client.stop()
await asyncio.sleep(5)
def main():
client = Client(
demisto.getParam("api_access_id"),
demisto.getParam("api_secret_key"),
demisto.getParam("tc_api_path"),
)
command = demisto.command()
demisto.info(f'Command being called is {command}')
commands = {
'test-module': module_test_command,
f'{INTEGRATION_COMMAND_NAME}-get-indicators': get_indicators_command,
f'{INTEGRATION_COMMAND_NAME}-get-owners': get_owners_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client)
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](client)
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
return_error(
f'Integration {INTEGRATION_NAME} Failed to execute {command} command. Error: {str(e)}'
)
def check_for_mirrors():
"""
Checks for newly created mirrors and updates the server accordingly
"""
integration_context = demisto.getIntegrationContext()
if integration_context.get('mirrors'):
mirrors = json.loads(integration_context['mirrors'])
for mirror in mirrors:
if not mirror['mirrored']:
demisto.info('Mirroring: {}'.format(
mirror['investigation_id']))
mirror = mirrors.pop(mirrors.index(mirror))
if mirror['mirror_to'] and mirror[
'mirror_direction'] and mirror['mirror_type']:
investigation_id = mirror['investigation_id']
mirror_type = mirror['mirror_type']
auto_close = mirror['auto_close']
direction = mirror['mirror_direction']
if isinstance(auto_close, str):
auto_close = bool(strtobool(auto_close))
demisto.mirrorInvestigation(
investigation_id,
'{}:{}'.format(mirror_type, direction), auto_close)
mirror['mirrored'] = True
mirrors.append(mirror)
else:
demisto.info('Could not mirror {}'.format(
mirror['investigation_id']))
set_to_latest_integration_context('mirrors', mirrors)
def main():
params = demisto.params()
feed_tags = argToList(params.get('feedTags'))
tlp_color = params.get('tlp_color')
command = demisto.command()
demisto.info(f'Command being called is {command}')
# Switch case
commands = {
'test-module': module_test_command,
'autofocus-get-indicators': get_indicators_command
}
try:
auto_focus_key_retriever = AutoFocusKeyRetriever(params.get('api_key'))
client = Client(api_key=auto_focus_key_retriever.key,
insecure=params.get('insecure'),
proxy=params.get('proxy'),
indicator_feeds=params.get('indicator_feeds'),
custom_feed_urls=params.get('custom_feed_urls'),
scope_type=params.get('scope_type'),
sample_query=params.get('sample_query'))
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client, feed_tags, tlp_color)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args(), feed_tags, tlp_color) # type: ignore
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
raise Exception(f'Error in {SOURCE_NAME} Integration [{e}]')
def main():
params = demisto.params()
client = Client(params.get('indicator_type'), params.get('api_token'),
params.get('services'), params.get('risk_rule'),
params.get('fusion_file_path'), params.get('insecure'),
params.get('polling_timeout'), params.get('proxy'),
params.get('threshold'))
command = demisto.command()
demisto.info('Command being called is {}'.format(command))
# Switch case
commands = {
'test-module': test_module,
'rf-feed-get-indicators': get_indicators_command,
'rf-feed-get-risk-rules': get_risk_rules_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client,
client.indicator_type)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args()) # type:ignore
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}] \n Traceback: {traceback.format_exc()}'
return_error(err_msg)
def update_incident(incident_id, assigned_user_mail, assigned_user_pretty_name,
status, severity, resolve_comment, unassign_user):
update_data = {}
if unassign_user and (assigned_user_mail or assigned_user_pretty_name):
raise ValueError(
"Can't provide both assignee_email/assignee_name and unassign_user"
)
elif unassign_user:
update_data['assigned_user_mail'] = 'none'
if assigned_user_mail:
update_data['assigned_user_mail'] = assigned_user_mail
if assigned_user_pretty_name:
update_data['assigned_user_pretty_name'] = assigned_user_pretty_name
if status:
update_data['status'] = status
if severity:
update_data['manual_severity'] = severity
if resolve_comment:
update_data['resolve_comment'] = resolve_comment
request_data = {'incident_id': incident_id, 'update_data': update_data}
demisto.info(json.dumps(request_data, indent=4))
http_request('POST',
'/incidents/update_incident/',
data={'request_data': request_data})
def get_versions(self, versions_filter: str):
"""
:param versions_filter: (str) contains the filter of the request.
:return: the result of the http request from the api
"""
demisto.info(f'fileters for version {versions_filter}')
return self._http_request(method='GET', url_suffix=URL_SUFFIX['VERSIONS'].format(versions_filter))
def main():
params = demisto.params()
args = demisto.args()
base_url = "https://rules.emergingthreats.net/"
client = Client(
base_url=base_url,
auth_code=params.get("auth_code"),
verify=not params.get("insecure", False),
proxy=params.get("proxy"),
)
command = demisto.command()
demisto.info("Command being called is {}".format(command))
# Switch case
try:
if command == "fetch-indicators":
indicators = fetch_indicators_command(client,
params.get("indicator_type"))
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
elif command == "test-module":
return_outputs(
module_test_command(client, params.get("indicator_type")))
elif command == "proofpoint-get-indicators":
readable_output, outputs, raw_response = get_indicators_command(
client, args)
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
return_error(
f"Error in {SOURCE_NAME} Integration - Encountered an issue with createIndicators"
if "failed to create" in
str(e) else f"Error in {SOURCE_NAME} Integration [{e}]")
def main():
# Write configure here
params = {k: v for k, v in demisto.params().items() if v is not None}
handle_proxy()
client = TAXIIClient(**params)
command = demisto.command()
demisto.info('Command being called is {command}'.format(command=command))
# Switch case
commands = {
'test-module': test_module,
'get-indicators': get_indicators_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client)
# we submit the indicators in batches
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
demisto.setLastRun({'time': client.last_taxii_run})
else:
readable_output, outputs, raw_response = commands[command](client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
err_msg = f'Error in {INTEGRATION_NAME} Integration [{e}]'
raise Exception(err_msg)
def find_indicators_by_time_frame(indicator_query: str, begin_time: datetime, end_time: datetime) -> list:
"""
Find indicators according to a query and begin time/end time.
Args:
indicator_query: The indicator query.
begin_time: The exclusive begin time.
end_time: The inclusive end time.
Returns:
Indicator query results from Demisto.
"""
if indicator_query:
indicator_query += ' and '
else:
indicator_query = ''
if begin_time:
tz_begin_time = datetime.strftime(begin_time, '%Y-%m-%dT%H:%M:%S %z')
indicator_query += f'sourcetimestamp:>"{tz_begin_time}"'
if end_time:
indicator_query += ' and '
if end_time:
tz_end_time = datetime.strftime(end_time, '%Y-%m-%dT%H:%M:%S %z')
indicator_query += f'sourcetimestamp:<="{tz_end_time}"'
demisto.info(f'Querying indicators by: {indicator_query}')
return find_indicators_loop(indicator_query)
def webhook_event_send_command(client: Client, args: dict):
auth_token = args.get("auth_token", "")
options: str = args.get("options", "")
free_json: str = args.get("json", "")
options_as_dict: dict = attribute_pairs_to_dict(options)
try:
demisto.info('start convert "options" argument to str')
if options_as_dict:
options_as_str: str = json.dumps(options_as_dict)
else:
options_as_str = free_json
demisto.info('finish convert "options" argument to str')
except Exception as e:
raise DemistoException(
f'There was a problem converting "json" to json. The reason is: {e}'
)
result = client.webhook_event_send(auth_token, options_as_str, free_json)
headers = [key.replace("_", " ") for key in [*result.keys()]]
readable_output = tableToMarkdown(
"Webhook event send:", result, headers=headers, headerTransform=pascalToSpace
)
return CommandResults(
readable_output=readable_output,
outputs_prefix="Rundeck.WebhookEvent",
outputs=result,
outputs_key_field="id",
)
def generate_incidents(last_run):
"""
Determines how many incidents to create and generates them
parameter: (number) last_run
The number of incidents generated in the last fetch
returns:
The number of incidents generated in the current call to fetch_incidents and the incidents themselves
"""
if last_run > 0 and last_run > MAX_NUM_OF_INCIDENTS:
demisto.info('last_run is greater than MAX_NUM_OF_INCIDENTS')
return 0, []
incidents = []
num_of_incidents_left_to_create = MAX_NUM_OF_INCIDENTS - last_run
if num_of_incidents_left_to_create > INCIDENTS_PER_MINUTE:
num_of_incident_to_create = INCIDENTS_PER_MINUTE
else:
num_of_incident_to_create = num_of_incidents_left_to_create
for _ in range(num_of_incident_to_create):
email, email_object = create_email()
incidents.append({
'name': email_object.get('Subject'),
'details': email.as_string(),
'occurred': email_object.get('Date'),
'type': INCIDENT_TYPE,
'rawJSON': json.dumps(email_object)
})
return num_of_incident_to_create, incidents
def docker_auth(image_name, verify_ssl=True, registry=DEFAULT_REGISTRY):
"""
Authenticate to the docker service. Return an authentication token if authentication is required.
"""
res = requests.get("https://{}/v2/".format(registry), headers=ACCEPT_HEADER,
timeout=TIMEOUT, verify=verify_ssl)
if res.status_code == 401: # need to authenticate
# defaults in case we fail for some reason
realm = "https://auth.docker.io/token"
service = "registry.docker.io"
# Shold contain header: Www-Authenticate
www_auth = res.headers.get('www-authenticate')
if www_auth:
parse_auth = parse_www_auth(www_auth)
if parse_auth:
realm, service = parse_auth
else:
demisto.info('Failed parsing www-authenticate header: {}'.format(www_auth))
else:
demisto.info('Failed extracting www-authenticate header from registry: {}, final url: {}'.format(
registry, res.url))
res = requests.get(
"{}?scope=repository:{}:pull&service={}".format(realm, image_name, service),
headers=ACCEPT_HEADER, timeout=TIMEOUT, verify=verify_ssl)
res.raise_for_status()
res_json = res.json()
return res_json.get('token')
else:
res.raise_for_status()
return None
def main():
params = demisto.params()
username = params.get('credentials').get('identifier')
password = params.get('credentials').get('password')
domain = params.get('domain')
# Remove trailing slash to prevent wrong URL path to service
server = params['url'].strip('/')
# Service base URL
base_url = server + '/api/v2/'
# Should we use SSL
use_ssl = not params.get('insecure', False)
# Remove proxy if not set to true in params
handle_proxy()
command = demisto.command()
client = Client(base_url, username, password, domain, verify=use_ssl)
demisto.info(f'Command being called is {command}')
commands = {
'test-module': test_module,
'tn-get-system-status': get_system_status,
'tn-get-package': get_package,
'tn-create-package': create_package,
'tn-list-packages': get_packages,
'tn-get-sensor': get_sensor,
'tn-list-sensors': get_sensors,
'tn-ask-question': ask_question,
'tn-get-question-metadata': get_question_metadata,
'tn-get-question-result': get_question_result,
'tn-create-saved-question': create_saved_question,
'tn-get-saved-question-metadata': get_saved_question_metadata,
'tn-get-saved-question-result': get_saved_question_result,
'tn-list-saved-questions': get_saved_questions,
'tn-create-action': create_action,
'tn-create-action-by-host': create_action_by_host,
'tn-get-action': get_action,
'tn-list-actions': get_actions,
'tn-create-saved-action': create_saved_action,
'tn-get-saved-action': get_saved_action,
'tn-list-saved-actions': get_saved_actions,
'tn-list-saved-actions-pending-approval': get_saved_actions_pending,
'tn-create-filter-based-group': create_filter_based_group,
'tn-create-manual-group': create_manual_group,
'tn-get-group': get_group,
'tn-list-groups': get_groups,
'tn-delete-group': delete_group
}
try:
if command in commands:
human_readable, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output=human_readable,
outputs=outputs,
raw_response=raw_response)
# Log exceptions
except Exception as e:
err_msg = f'Error in Tanium v2 Integration [{e}]'
return_error(err_msg, error=e)
def write(message):
"""
Writes a log message to the Demisto server.
Args:
message: The log message to write
"""
demisto.info(message)
def get_last_run() -> str:
""" Gets last run time in timestamp
Returns:
last run in timestamp, or '' if no last run
"""
if last_run := demisto.getIntegrationContext().get('last_run'):
demisto.info(f'get last_run: {last_run}')
params = f'date:{last_run}+'
def base_dn_verified(base_dn):
# serch AD with a simple query to test base DN is configured correctly
try:
search("(objectClass=user)", base_dn, size_limit=1)
except Exception as e:
demisto.info(str(e))
return False
return True
def process_attachments(attachCIDs="",
attachIDs="",
attachNames="",
manualAttachObj=None):
if manualAttachObj is None:
manualAttachObj = []
file_entries_for_attachments = [] # type: list
attachments_names = [] # type: list
if attachIDs:
file_entries_for_attachments = attachIDs if isinstance(
attachIDs, list) else attachIDs.split(",")
if attachNames:
attachments_names = attachNames if isinstance(
attachNames, list) else attachNames.split(",")
else:
for att_id in file_entries_for_attachments:
att_name = demisto.getFilePath(att_id)['name']
if isinstance(att_name, list):
att_name = att_name[0]
attachments_names.append(att_name)
if len(file_entries_for_attachments) != len(attachments_names):
raise Exception(
"attachIDs and attachNames lists should be the same length")
attachments = collect_manual_attachments(manualAttachObj)
if attachCIDs:
file_entries_for_attachments_inline = attachCIDs if isinstance(
attachCIDs, list) else attachCIDs.split(",")
for att_id_inline in file_entries_for_attachments_inline:
try:
file_info = demisto.getFilePath(att_id_inline)
except Exception as ex:
demisto.info("EWS error from getFilePath: {}".format(ex))
raise Exception("entry %s does not contain a file" %
att_id_inline)
att_name_inline = file_info["name"]
with open(file_info["path"], 'rb') as f:
attachments.append(
FileAttachment(content=f.read(),
name=att_name_inline,
is_inline=True,
content_id=att_name_inline))
for i in range(0, len(file_entries_for_attachments)):
entry_id = file_entries_for_attachments[i]
attachment_name = attachments_names[i]
try:
res = demisto.getFilePath(entry_id)
except Exception as ex:
raise Exception("entry {} does not contain a file: {}".format(
entry_id, str(ex)))
file_path = res["path"]
with open(file_path, 'rb') as f:
attachments.append(
FileAttachment(content=f.read(), name=attachment_name))
return attachments, attachments_names
def fetch_incidents(proxies):
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch, _ = parse_date_range(date_range=FETCH_TIME,
date_format='%Y-%m-%dT%H:%M:%S.%f',
utc=False,
to_timestamp=False)
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if timestamp: get the last fetch to the correct format of timestamp
if 'Timestamp' in TIME_METHOD:
last_fetch = get_timestamp_first_fetch(last_fetch)
last_fetch_timestamp = last_fetch
# if method is simple date - convert the date string to datetime
elif 'Simple-Date' == TIME_METHOD:
last_fetch = parse(str(last_fetch))
last_fetch_timestamp = int(last_fetch.timestamp() * 1000)
# if last_fetch is set and we are in a "Timestamp" method - than the last_fetch_timestamp is the last_fetch.
else:
last_fetch_timestamp = last_fetch
es = elasticsearch_builder(proxies)
query = QueryString(query=FETCH_QUERY + " AND " + TIME_FIELD + ":*")
# Elastic search can use epoch timestamps (in milliseconds) as date representation regardless of date format.
search = Search(using=es, index=FETCH_INDEX).filter(
{'range': {
TIME_FIELD: {
'gt': last_fetch_timestamp
}
}})
search = search.sort({TIME_FIELD: {
'order': 'asc'
}})[0:FETCH_SIZE].query(query)
response = search.execute().to_dict()
_, total_results = get_total_results(response)
incidents = [] # type: List
if total_results > 0:
if 'Timestamp' in TIME_METHOD:
incidents, last_fetch = results_to_incidents_timestamp(
response, last_fetch)
demisto.setLastRun({'time': last_fetch})
else:
incidents, last_fetch = results_to_incidents_datetime(
response, last_fetch)
demisto.setLastRun({'time': str(last_fetch)})
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.incidents(incidents)
def get_access_token():
integration_context = demisto.getIntegrationContext()
access_token = integration_context.get('access_token')
valid_until = integration_context.get('valid_until')
if access_token and valid_until:
if epoch_seconds() < valid_until:
return access_token
headers = {'Accept': 'application/json'}
dbot_response = requests.post(
TOKEN_RETRIEVAL_URL,
headers=headers,
data=json.dumps({
'app_name': APP_NAME,
'registration_id': AUTH_ID,
'encrypted_token': get_encrypted(TENANT_ID, ENC_KEY)
}),
verify=USE_SSL
)
if dbot_response.status_code not in {200, 201}:
msg = 'Error in authentication. Try checking the credentials you entered.'
try:
demisto.info('Authentication failure from server: {} {} {}'.format(
dbot_response.status_code, dbot_response.reason, dbot_response.text))
err_response = dbot_response.json()
server_msg = err_response.get('message')
if not server_msg:
title = err_response.get('title')
detail = err_response.get('detail')
if title:
server_msg = f'{title}. {detail}'
if server_msg:
msg += ' Server message: {}'.format(server_msg)
except Exception as ex:
demisto.error('Failed parsing error response - Exception: {}'.format(ex))
raise Exception(msg)
try:
gcloud_function_exec_id = dbot_response.headers.get('Function-Execution-Id')
demisto.info(f'Google Cloud Function Execution ID: {gcloud_function_exec_id}')
parsed_response = dbot_response.json()
except ValueError:
raise Exception(
'There was a problem in retrieving an updated access token.\n'
'The response from the Demistobot server did not contain the expected content.'
)
access_token = parsed_response.get('access_token')
expires_in = parsed_response.get('expires_in', 3595)
time_now = epoch_seconds()
time_buffer = 5 # seconds by which to shorten the validity period
if expires_in - time_buffer > 0:
# err on the side of caution with a slightly shorter access token validity period
expires_in = expires_in - time_buffer
demisto.setIntegrationContext({
'access_token': access_token,
'valid_until': time_now + expires_in
})
return access_token
def main() -> None:
"""main function, parses params and runs command functions
:return:
:rtype:
"""
# INTEGRATION DEVELOPER TIP
# You can use functions such as ``demisto.debug()``, ``demisto.info()``,
# etc. to print information in the XSOAR server log. You can set the log
# level on the server configuration
# See: https://xsoar.pan.dev/docs/integrations/code-conventions#logging
command = demisto.command()
args = demisto.args()
demisto.info(f'Command running: {demisto.command()}')
try:
client = create_client()
xm = XM(client)
# commands dict
# key - command key
# value - command execution function that get two params:
# 1) XM object
# 2) args dict
# return value - CommandResults
commandsDict = {
"test-module":
test_module_command_internal, # This is the call made when pressing the integration Test button.
"xmcyber-f-incidents":
fetch_incidents_command, # for debugging of fetch incidents
# XM Cyber Command list
# xmcyber-command-name: function_command
"xmcyber-version-get": get_version_command,
"xmcyber-version-supported": is_xm_version_supported_command,
"xmcyber-affected-critical-assets-list":
affected_critical_assets_list_command,
"xmcyber-affected-entities-list": affected_entities_list_command,
# Common commands
"ip": ip_command,
"xmcyber-hostname": hostname_command
}
if command == 'fetch-incidents':
xm.is_fetch_incidents = True
fetch_incidents_command(xm, args)
elif command in commandsDict:
return_results(commandsDict[command](xm, args))
else:
raise Exception("Unsupported command: " + command)
# Log exceptions and return errors
except Exception as e:
demisto.error(traceback.format_exc()) # print the traceback
return_error(
f'Failed to execute {demisto.command()} command.\nError:\n{str(e)}\n'
f'Traceback:\n{traceback.format_exc()}')
def main() -> None:
"""main function, parses params and runs command functions
:return:
:rtype:
"""
polaris_account = demisto.params().get('polaris_account', False)
polaris_domain_name = "my.rubrik.com"
polaris_base_url = f"https://{polaris_account}.{polaris_domain_name}/api"
headers = {
'Content-Type': 'application/json;charset=UTF-8',
'Accept': 'application/json, text/plain'
}
client = Client(base_url=polaris_base_url, headers=headers, verify=False)
max_fetch = demisto.params().get('max_fetch')
max_fetch = int(demisto.params().get('max_fetch')) if (
max_fetch and max_fetch.isdigit()) else 50
max_fetch = max(min(200, max_fetch), 1)
demisto.info(f'Command being called is {demisto.command()}')
try:
if demisto.command() == 'test-module':
# This is the call made when pressing the integration Test button.
result = test_module(client)
return_results(result)
elif demisto.command() == 'fetch-incidents':
current_time, incidents = fetch_incidents(client, max_fetch)
# A blank current_time is returned when it has not been 5 minutes
# since the last fetch
if current_time != "":
demisto.setLastRun({'last_fetch': current_time})
demisto.incidents(incidents)
elif demisto.command() == 'rubrik-radar-analysis-status':
return_results(
rubrik_radar_analysis_status_command(client, demisto.args()))
elif demisto.command() == 'rubrik-sonar-sensitive-hits':
return_results(
rubrik_sonar_sensitive_hits_command(client, demisto.args()))
elif demisto.command() == 'rubrik-cdm-cluster-location':
return_results(
rubrik_cdm_cluster_location_command(client, demisto.args()))
elif demisto.command() == 'rubrik-cdm-cluster-connection-state':
return_results(
rubrik_cdm_cluster_connection_state_command(
client, demisto.args()))
# Log exceptions and return errors
except Exception as e:
demisto.error(traceback.format_exc()) # print the traceback
return_error(
f'Failed to execute {demisto.command()} command.\nError:\n{str(e)}'
)
async def fetch_event(self, initial_offset: int = 0, event_type: str = '') -> AsyncGenerator[Dict, None]:
"""Retrieves events from a CrowdStrike Falcon stream starting from given offset.
Args:
initial_offset (int): Stream offset to start the fetch from.
event_type (str): Stream event type to fetch.
Yields:
AsyncGenerator[Dict, None]: Event fetched from the stream.
"""
while True:
demisto.debug('Fetching event')
event = Event()
create_task(self._discover_refresh_stream(event))
demisto.debug('Waiting for stream discovery or refresh')
await event.wait()
demisto.debug('Done waiting for stream discovery or refresh')
events_fetched = 0
new_lines_fetched = 0
last_fetch_stats_print = datetime.utcnow()
async with ClientSession(
connector=TCPConnector(ssl=self.verify_ssl),
headers={'Authorization': f'Token {self.session_token}'},
trust_env=self.proxy,
timeout=None
) as session:
try:
integration_context = demisto.getIntegrationContext()
offset = integration_context.get('offset', 0) or initial_offset
demisto.debug(f'Starting to fetch from offset {offset} events of type {event_type}')
async with session.get(
self.data_feed_url,
params={'offset': offset, 'eventType': event_type},
timeout=None
) as res:
demisto.debug(f'Fetched event: {res.content}')
async for line in res.content:
stripped_line = line.strip()
if stripped_line:
events_fetched += 1
try:
yield json.loads(stripped_line)
except json.decoder.JSONDecodeError:
demisto.debug(f'Failed decoding event (skipping it) - {str(stripped_line)}')
else:
new_lines_fetched += 1
if last_fetch_stats_print + timedelta(minutes=1) <= datetime.utcnow():
demisto.info(
f'Fetched {events_fetched} events and'
f' {new_lines_fetched} new lines'
f' from the stream in the last minute.')
events_fetched = 0
new_lines_fetched = 0
last_fetch_stats_print = datetime.utcnow()
except Exception as e:
demisto.debug(f'Failed to fetch event: {e} - Going to sleep for 10 seconds and then retry -'
f' {traceback.format_exc()}')
await sleep(10)
def main():
"""
PARSE AND VALIDATE INTEGRATION PARAMS
"""
params = demisto.params()
access_token = params.get('access_token')
environment = params.get('environment')
verify_certificate = not params.get('insecure', False)
first_fetch_time = params.get('first_fetch', '3 days').strip()
proxy = params.get('proxy', False)
base_url = f'https://{environment}.cyberint.io/alert/'
demisto.info(f'Command being called is {demisto.command()}')
try:
client = Client(base_url=base_url,
verify_ssl=verify_certificate,
access_token=access_token,
proxy=proxy)
if demisto.command() == 'test-module':
result = test_module(client)
return_results(result)
elif demisto.command() == 'fetch-incidents':
fetch_environment = argToList(params.get('fetch_environment', ''))
fetch_status = params.get('fetch_status', [])
fetch_type = params.get('fetch_type', [])
fetch_severity = params.get('fetch_severity', [])
max_fetch = int(params.get('max_fetch', '50'))
next_run, incidents = fetch_incidents(client, demisto.getLastRun(),
first_fetch_time,
fetch_severity, fetch_status,
fetch_type,
fetch_environment, max_fetch)
demisto.setLastRun(next_run)
demisto.incidents(incidents)
elif demisto.command() == 'cyberint-alerts-fetch':
return_results(
cyberint_alerts_fetch_command(client, demisto.args()))
elif demisto.command() == 'cyberint-alerts-status-update':
return_results(
cyberint_alerts_status_update(client, demisto.args()))
except Exception as e:
if 'Invalid token or token expired' in str(e):
error_message = 'Error verifying access token and / or environment, make sure the ' \
'configuration parameters are correct.'
elif 'datetime' in str(e).lower():
error_message = 'Invalid time specified, ' \
'make sure the arguments are correctly formatted and are not ' \
'earlier than 2020 or later than the current time.'
elif 'Unauthorized alerts requested' in str(e):
error_message = 'Some of the alerts selected to update are either blocked or not found.'
else:
error_message = f'Failed to execute {demisto.command()} command. Error: {str(e)}'
return_error(error_message)
def base_dn_verified(base_dn):
# serch AD with a simple query to test base DN is configured correctly
try:
search(
"(objectClass=user)",
base_dn,
size_limit=1
)
except Exception as e:
demisto.info(str(e))
return False
return True
def get_access_token():
integration_context = demisto.getIntegrationContext()
access_token = integration_context.get('access_token')
stored = integration_context.get('stored')
if access_token and stored:
if epoch_seconds() - stored < 60 * 60 - 30:
return access_token
headers = {
'Authorization': AUTH_ID,
'Accept': 'application/json'
}
dbot_response = requests.get(
TOKEN_RETRIEVAL_URL,
headers=headers,
params={'token': get_encrypted(TOKEN, ENC_KEY)},
verify=USE_SSL
)
if dbot_response.status_code not in {200, 201}:
msg = 'Error in authentication. Try checking the credentials you entered.'
try:
demisto.info('Authentication failure from server: {} {} {}'.format(
dbot_response.status_code, dbot_response.reason, dbot_response.text))
err_response = dbot_response.json()
server_msg = err_response.get('message')
if server_msg:
msg += ' Server message: {}'.format(server_msg)
except Exception as ex:
demisto.error('Failed parsing error response: [{}]. Exception: {}'.format(err_response.content, ex))
raise Exception(msg)
try:
parsed_response = dbot_response.json()
except ValueError:
raise Exception(
'There was a problem in retrieving an updated access token.\n'
'The response from the Demistobot server did not contain the expected content.'
)
access_token = parsed_response.get('access_token')
api_url = parsed_response.get('url')
token = parsed_response.get('token')
demisto.setIntegrationContext({
'access_token': access_token,
'stored': epoch_seconds(),
'api_url': api_url,
'token': token
})
return access_token
def fetch_incidents():
params = demisto.params()
user_key = params.get('queryUserKey')
user_key = user_key if user_key else ADMIN_EMAIL
query = '' if params['query'] is None else params['query']
last_run = demisto.getLastRun()
last_fetch = last_run.get('time')
# handle first time fetch
if last_fetch is None:
last_fetch = datetime.datetime.now() - datetime.timedelta(days=1)
else:
last_fetch = datetime.datetime.strptime(
last_fetch, '%Y-%m-%dT%H:%M:%SZ')
current_fetch = last_fetch
credentials = get_credentials(
['https://www.googleapis.com/auth/gmail.readonly', ], user_key)
service = discovery.build('gmail', 'v1', credentials=credentials)
query += last_fetch.strftime(' after:%Y/%m/%d')
LOG('GMAIL: fetch parameters:\nuser: %s\nquery=%s\nfetch time: %s' %
(user_key, query, last_fetch, ))
result = service.users().messages().list(
userId=user_key, maxResults=100, q=query).execute()
incidents = []
# so far, so good
LOG('GMAIL: possible new incidents are %s' % (result, ))
for msg in result.get('messages', []):
msg_result = service.users().messages().get(
id=msg['id'], userId=user_key).execute()
incident = mail_to_incident(msg_result, service, user_key)
temp_date = datetime.datetime.strptime(
incident['occurred'], '%Y-%m-%dT%H:%M:%SZ')
# update last run
if temp_date > last_fetch:
last_fetch = temp_date + datetime.timedelta(seconds=1)
# avoid duplication due to weak time query
if temp_date > current_fetch:
incidents.append(incident)
demisto.info('extract {} incidents'.format(len(incidents)))
demisto.setLastRun({'time': last_fetch.isoformat().split('.')[0] + 'Z'})
return incidents
def main():
if demisto.args().get('use_system_proxy') == 'no':
del os.environ['HTTP_PROXY']
del os.environ['HTTPS_PROXY']
del os.environ['http_proxy']
del os.environ['https_proxy']
verify_ssl = demisto.args().get('trust_any_certificate') != 'yes'
docker_full_name = demisto.args()['input']
registry = DEFAULT_REGISTRY
image_name_tag = docker_full_name
if docker_full_name.count('/') > 1:
registry, image_name_tag = docker_full_name.split('/', 1)
try:
split = image_name_tag.split(':')
image_name = split[0]
tag = 'latest'
if len(split) > 1:
tag = split[1]
if tag is None:
tag = 'latest'
auth_token = docker_auth(image_name, verify_ssl, registry)
headers = ACCEPT_HEADER.copy()
if auth_token:
headers['Authorization'] = "Bearer {}".format(auth_token)
res = requests.get("https://{}/v2/{}/manifests/{}".format(registry, image_name, tag),
headers=headers, timeout=TIMEOUT, verify=verify_ssl)
res.raise_for_status()
layers = res.json().get('layers')
if not layers:
raise ValueError("No 'layers' found in json response: {}".format(res.content))
layer_min = docker_min_layer(layers)
headers['Range'] = "bytes=0-99"
res = requests.get("https://{}/v2/{}/blobs/{}".format(registry, image_name, layer_min['digest']),
headers=headers, timeout=TIMEOUT, verify=verify_ssl)
res.raise_for_status()
expected_len = min([100, layer_min['size']])
cont_len = len(res.content)
demisto.info("Docker image check [{}] downloaded layer content of len: {}".format(docker_full_name, cont_len))
if cont_len < expected_len:
raise ValueError('Content returned is shorter than expected length: {}. Content: {}'.format(expected_len,
res.content))
demisto.results('ok')
except Exception as ex:
return_error("Failed verifying: {}. Err: {}".format(docker_full_name, str(ex)))
def create_contact():
assert conn is not None
args = demisto.args()
object_classes = ["top", "person", "organizationalPerson", "contact"]
contact_dn = args.get('contact-dn')
# set contact attributes
attributes: Dict = {}
if args.get('custom-attributes'):
try:
attributes = json.loads(args['custom-attributes'])
except Exception as e:
demisto.info(str(e))
raise Exception(
'Failed to parse custom attributes argument. Please see an example of this argument in the argument.'
)
# set common user attributes
if args.get('diaply-name'):
attributes['displayName'] = args['diaply-name']
if args.get('description'):
attributes['description'] = args['description']
if args.get('email'):
attributes['mail'] = args['email']
if args.get('telephone-number'):
attributes['telephoneNumber'] = args['telephone-number']
if args.get('title'):
attributes['title'] = args['title']
# add contact
success = conn.add(contact_dn, object_classes, attributes)
if not success:
raise Exception("Failed to create contact")
demisto_entry = {
'ContentsFormat': formats['text'],
'Type': entryTypes['note'],
'Contents': "Created contact with DN: {}".format(contact_dn)
}
demisto.results(demisto_entry)
def create_user():
assert conn is not None
args = demisto.args()
object_classes = ["top", "person", "organizationalPerson", "user"]
user_dn = args.get('user-dn')
username = args.get("username")
password = args.get("password")
custome_attributes = args.get('custom-attributes')
attributes = {
"samAccountName": username
}
# set common user attributes
if args.get('display-name'):
attributes['displayName'] = args['display-name']
if args.get('description'):
attributes['description'] = args['description']
if args.get('email'):
attributes['mail'] = args['email']
if args.get('telephone-number'):
attributes['telephoneNumber'] = args['telephone-number']
if args.get('title'):
attributes['title'] = args['title']
# set user custome attributes
if custome_attributes:
try:
custome_attributes = json.loads(custome_attributes)
except Exception as e:
demisto.info(str(e))
raise Exception(
"Failed to parse custom attributes argument. Please see an example of this argument in the description."
)
for attribute_name, attribute_value in custome_attributes.items():
# can run default attribute stting
attributes[attribute_name] = attribute_value
# add user
success = conn.add(user_dn, object_classes, attributes)
if not success:
raise Exception("Failed to create user")
# set user password
success = conn.extend.microsoft.modify_password(user_dn, password)
if not success:
raise Exception("Failed to reset user password")
# enable user and expire password
modification = {
# enable user
'userAccountControl': [('MODIFY_REPLACE', NORMAL_ACCOUNT)],
# set to 0, to force password change on next login
"pwdLastSet": [('MODIFY_REPLACE', "0")]
}
modify_object(user_dn, modification)
demisto_entry = {
'ContentsFormat': formats['text'],
'Type': entryTypes['note'],
'Contents': "Created user with DN: {}".format(user_dn)
}
demisto.results(demisto_entry)
def main():
''' INSTANCE CONFIGURATION '''
SERVER_IP = demisto.params().get('server_ip')
USERNAME = demisto.params().get('credentials')['identifier']
PASSWORD = demisto.params().get('credentials')['password']
DEFAULT_BASE_DN = demisto.params().get('base_dn')
SECURE_CONNECTION = demisto.params().get('secure_connection')
DEFAULT_PAGE_SIZE = int(demisto.params().get('page_size'))
NTLM_AUTH = demisto.params().get('ntlm')
UNSECURE = demisto.params().get('unsecure', False)
PORT = demisto.params().get('port')
if PORT:
# port was configured, cast to int
PORT = int(PORT)
try:
server = initialize_server(SERVER_IP, PORT, SECURE_CONNECTION, UNSECURE)
except Exception as e:
return_error(str(e))
return
global conn
if NTLM_AUTH:
# intialize connection to LDAP server with NTLM authentication
# user example: domain\user
domain_user = SERVER_IP + '\\' + USERNAME if '\\' not in USERNAME else USERNAME
conn = Connection(server, user=domain_user, password=PASSWORD, authentication=NTLM)
else:
# here username should be the user dn
conn = Connection(server, user=USERNAME, password=PASSWORD)
# bind operation is the “authenticate” operation.
try:
# open socket and bind to server
if not conn.bind():
message = "Failed to bind to server. Please validate the credentials configured correctly.\n{}".format(
json.dumps(conn.result))
demisto.info(message)
return_error(message)
return
except Exception as e:
exc_msg = str(e)
demisto.info("Failed bind to: {}:{}. {}: {}".format(SERVER_IP, PORT, type(e), exc_msg
+ "\nTrace:\n{}".format(traceback.format_exc())))
message = "Failed to access LDAP server. Please validate the server host and port are configured correctly"
if 'ssl wrapping error' in exc_msg:
message = "Failed to access LDAP server. SSL error."
if not UNSECURE:
message += ' Try using: "Trust any certificate" option.'
demisto.info(message)
return_error(message)
return
demisto.info('Established connection with AD LDAP server')
if not base_dn_verified(DEFAULT_BASE_DN):
message = "Failed to verify the base DN configured for the instance.\n" \
"Last connection result: {}\n" \
"Last error from LDAP server: {}".format(json.dumps(conn.result), json.dumps(conn.last_error))
demisto.info(message)
return_error(message)
return
demisto.info('Verfied base DN "{}"'.format(DEFAULT_BASE_DN))
''' COMMAND EXECUTION '''
try:
if demisto.command() == 'test-module':
if conn.user == '':
# Empty response means you have no authentication status on the server, so you are an anonymous user.
raise Exception("Failed to authenticate user")
demisto.results('ok')
if demisto.command() == 'ad-search':
free_search(DEFAULT_BASE_DN, DEFAULT_PAGE_SIZE)
if demisto.command() == 'ad-expire-password':
expire_user_password(DEFAULT_BASE_DN)
if demisto.command() == 'ad-set-new-password':
set_user_password(DEFAULT_BASE_DN)
if demisto.command() == 'ad-unlock-account':
unlock_account(DEFAULT_BASE_DN)
if demisto.command() == 'ad-disable-account':
disable_user(DEFAULT_BASE_DN)
if demisto.command() == 'ad-enable-account':
enable_user(DEFAULT_BASE_DN)
if demisto.command() == 'ad-remove-from-group':
remove_member_from_group(DEFAULT_BASE_DN)
if demisto.command() == 'ad-add-to-group':
add_member_to_group(DEFAULT_BASE_DN)
if demisto.command() == 'ad-create-user':
create_user()
if demisto.command() == 'ad-delete-user':
delete_user()
if demisto.command() == 'ad-update-user':
update_user(DEFAULT_BASE_DN)
if demisto.command() == 'ad-modify-computer-ou':
modify_computer_ou(DEFAULT_BASE_DN)
if demisto.command() == 'ad-create-contact':
create_contact()
if demisto.command() == 'ad-update-contact':
update_contact()
if demisto.command() == 'ad-get-user':
search_users(DEFAULT_BASE_DN, DEFAULT_PAGE_SIZE)
if demisto.command() == 'ad-get-computer':
search_computers(DEFAULT_BASE_DN, DEFAULT_PAGE_SIZE)
if demisto.command() == 'ad-get-group-members':
search_group_members(DEFAULT_BASE_DN, DEFAULT_PAGE_SIZE)
except Exception as e:
message = "{}\nLast connection result: {}\nLast error from LDAP server: {}".format(
str(e), json.dumps(conn.result), conn.last_error)
demisto.info(message)
return_error(message)
return
finally:
# disconnect and close the connection
conn.unbind()
