def sixgill_get_indicators_command():
max_indicators = get_limit(
demisto.args().get('maxIndicators', MAX_INDICATORS), MAX_INDICATORS)
sixgill_darkfeed_client = SixgillFeedClient(
demisto.params()['client_id'],
demisto.params()['client_secret'], CHANNEL_CODE, FeedStream.DARKFEED,
demisto, max_indicators, SESSION, VERIFY)
bundle = sixgill_darkfeed_client.get_bundle()
sixgill_darkfeed_client.commit_indicators()
num_of_indicators = 0
for stix_item in bundle.get("objects"):
if is_indicator(stix_item):
num_of_indicators += 1
if stix_item.get("sixgill_severity"):
stix_item['score'] = to_demisto_score(
stix_item.get("sixgill_severity", 0))
human_readable = f"# Fetched {num_of_indicators} DarkFeed indicators"
bundle_id = bundle.get("id", "bundle")
entry = fileResult(f'{bundle_id}.json', json.dumps(bundle),
entryTypes['entryInfoFile'])
entry["HumanReadable"] = human_readable
entry["ContentsFormat"] = formats["markdown"]
demisto.results(entry)
def fetch_indicators_command(client: SixgillFeedClient,
limit: int = 0,
get_indicators_mode: bool = False,
tags: list = []):
bundle = client.get_bundle()
indicators_to_create: List = []
indicator_values_set: Set = set()
for stix_indicator in bundle.get("objects"):
if is_indicator(stix_indicator):
demisto_indicators = stix2_to_demisto_indicator(
stix_indicator, demisto, tags)
for indicator in demisto_indicators:
if indicator.get("value") not in indicator_values_set:
indicator_values_set.add(indicator.get("value"))
indicators_to_create.append(indicator)
if get_indicators_mode and len(indicators_to_create) == limit:
break
if not get_indicators_mode:
client.commit_indicators()
return indicators_to_create
def main():
max_indicators = get_limit(
demisto.params().get("maxIndicators", MAX_INDICATORS), MAX_INDICATORS)
SESSION.proxies = handle_proxy()
client = SixgillFeedClient(demisto.params()["client_id"],
demisto.params()["client_secret"],
CHANNEL_CODE,
FeedStream.DVEFEED,
bulk_size=max_indicators,
session=SESSION,
logger=demisto,
verify=VERIFY)
command = demisto.command()
demisto.info(f"Command being called is {command}")
tags = argToList(demisto.params().get("feedTags", []))
tlp_color = demisto.params().get("tlp_color")
commands: Dict[str, Callable] = {
"test-module": module_command_test,
"cybersixgill-get-indicators": get_indicators_command
}
try:
if demisto.command() == "fetch-indicators":
indicators = fetch_indicators_command(client,
tags=tags,
tlp_color=tlp_color)
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as err:
demisto.error(traceback.format_exc())
return_error(
f"Error failed to execute {demisto.command()}, error: [{err}]")
def test_feed_tags(mocker):
"""
Given:
- feedTags parameter
When:
- Executing fetch command on feed
Then:
- Validate the tags supplied are added to the tags list in addition to the tags that were there before
"""
global bundle_index
global submitted_indicators
mocker.patch.object(demisto, 'params', return_value=init_params())
mocker.patch('requests.sessions.Session.send', new=mocked_request)
from Sixgill_Darkfeed import fetch_indicators_command
from sixgill.sixgill_feed_client import SixgillFeedClient
from sixgill.sixgill_constants import FeedStream
client = SixgillFeedClient("client_id", "client_secret", "some_channel",
FeedStream.DARKFEED, demisto, 1000)
output = fetch_indicators_command(client, tags=['tag1', 'tag2'])
assert all(item in output[0]['fields']['tags']
for item in ['tag1', 'tag2'])
assert any(item in output[0]['fields']['tags']
for item in ['compromised', 'ip', 'url'])
def main():
max_indicators = get_limit(
demisto.params().get('maxIndicators', MAX_INDICATORS), MAX_INDICATORS)
SESSION.proxies = handle_proxy()
client = SixgillFeedClient(demisto.params()['client_id'],
demisto.params()['client_secret'], CHANNEL_CODE,
FeedStream.DARKFEED, demisto, max_indicators,
SESSION, VERIFY)
command = demisto.command()
demisto.info(f'Command being called is {command}')
commands: Dict[str, Callable] = {
'test-module': test_module_command,
'sixgill-get-indicators': get_indicators_command
}
try:
if demisto.command() == 'fetch-indicators':
indicators = fetch_indicators_command(client)
for b in batch(indicators, batch_size=2000):
demisto.createIndicators(b)
else:
readable_output, outputs, raw_response = commands[command](
client, demisto.args())
return_outputs(readable_output, outputs, raw_response)
except Exception as e:
demisto.error(traceback.format_exc())
return_error(
f'Error failed to execute {demisto.command()}, error: [{e}]')
def test_fetch_indicators_command(mocker):
global bundle_index
global submitted_indicators
mocker.patch.object(demisto, "params", return_value=init_params())
mocker.patch("requests.sessions.Session.send", new=mocked_request)
from CybersixgillDVEFeed import fetch_indicators_command
from sixgill.sixgill_feed_client import SixgillFeedClient
from sixgill.sixgill_constants import FeedStream
client = SixgillFeedClient(
demisto.params()["client_id"],
demisto.params()["client_secret"],
"some_channel",
FeedStream.DVEFEED,
demisto,
1000,
)
output = fetch_indicators_command(client)
bundle_index = 0
submitted_indicators = 0
assert output == expected_ioc_output
def fetch_indicators_command(client: SixgillFeedClient,
limit: int = 0,
get_indicators_mode: bool = False):
bundle = client.get_bundle()
indicators_to_create: List = []
for stix_indicator in bundle.get("objects"):
if is_indicator(stix_indicator):
demisto_indicators = stix2_to_demisto_indicator(
stix_indicator, demisto)
indicators_to_create.extend(demisto_indicators)
if get_indicators_mode and len(indicators_to_create) == limit:
break
if not get_indicators_mode:
client.commit_indicators()
return indicators_to_create
def test_get_indicators_command(mocker):
global bundle_index
global submitted_indicators
mocker.patch.object(demisto, 'params', return_value=init_params())
mocker.patch('requests.sessions.Session.send', new=mocked_request)
from Sixgill_Darkfeed import get_indicators_command
from sixgill.sixgill_feed_client import SixgillFeedClient
from sixgill.sixgill_constants import FeedStream
client = SixgillFeedClient("client_id", "client_secret", "some_channel",
FeedStream.DARKFEED, demisto, 1000)
output = get_indicators_command(client, {"limit": 10})
bundle_index = 0
submitted_indicators = 0
assert output[2] == expected_ioc_output
def test_feed_tags_and_tlp_color(mocker, tlp_color):
"""
Given:
- feedTags parameter
When:
- Executing fetch command on feed
Then:
- Validate the tags supplied are added to the tags list in addition to the tags that were there before
"""
global bundle_index
global submitted_indicators
mocker.patch.object(demisto, "params", return_value=init_params())
mocker.patch("requests.sessions.Session.send", new=mocked_request)
from CybersixgillDVEFeed import fetch_indicators_command
from sixgill.sixgill_feed_client import SixgillFeedClient
from sixgill.sixgill_constants import FeedStream
client = SixgillFeedClient(
demisto.params()["client_id"],
demisto.params()["client_secret"],
"some_channel",
FeedStream.DVEFEED,
demisto,
1000,
)
output = fetch_indicators_command(client,
tags=["tag1", "tag2"],
tlp_color=tlp_color)
assert all(item in output[0]["fields"]["tags"]
for item in ["tag1", "tag2"])
if tlp_color:
assert output[0]["fields"]["trafficlightprotocol"] == tlp_color
else:
assert not output[0]["fields"].get("trafficlightprotocol")
bundle_index -= 1
def on_poll(self, param):
"""This method ingest/update/delete the Sixgill Darkfeed Intelligence
Arguments:
param - paramter of the SixgillDarkfeed configuration
Returns:
action_result status -- Returns the 'action_result' status
"""
self._connector.debug_print(param)
action_result = ActionResult(dict(param))
self._connector.add_action_result(action_result)
# Setting up the limit to the SixgillFeedClient which defaults to 2000
if CONTAINER_COUNT in param and param.get(CONTAINER_COUNT) < 2000:
self._limit = param.get(CONTAINER_COUNT)
try:
darkfeed_client = SixgillFeedClient(
self._sixgill_client_id,
self._sixgill_api_secret_key,
self._sixgill_phantom_channel_id,
FeedStream.DARKFEED,
bulk_size=self._limit,
)
indicator_object = darkfeed_client.get_bundle().get("objects")
except Exception as e:
err = self._connector._get_error_message_from_exception(e)
return action_result.set_status(phantom.APP_ERROR, err)
sixgill_container = SixgillContainer(self._connector)
sixgill_artifact = SixgillArtifact(self._connector)
sixgill_utils = SixgillUtils(self._connector)
self._connector.save_progress(
"Ingesting Sixgill Darkfeed Intelligence ...")
failuer_indicator_list = []
if len(indicator_object) > 2:
for indicator in indicator_object:
if sixgill.sixgill_utils.is_indicator(indicator):
self._indicator_id = sixgill_utils.sixgill_delimit_id(
indicator.get(SIXGILL_FEED_ID))
try:
indicator_list = self._sixgill_get_sixgill_pattern_type(
indicator)
except:
self._connector.debug_print(
"Error occurred while processing sixgill indicator"
)
container = sixgill_container.prepare_container(indicator)
# Search the phantom if the indicator from API does exist in the phantom SOAR platform
# Retry search functionality for 5 times in case of a network issue
for count in range(int(TRY_AGAIN)):
try:
feed_dict = sixgill_utils.search_feed(
self._indicator_id)
if len(feed_dict) > 0:
break
except:
self._connector.debug_print(
"Error occurred while searching for the feed")
if feed_dict:
try:
# If the indicator includes a "revoked" = true flag, the indicator will get deleted from the phantom SOAR platform
sixgill_utils.delete_event(feed_dict, indicator,
sixgill_container,
sixgill_artifact,
self._verify_ssl)
# Update the event and the artifact if the source ID of the indicator exists in phantom SOAR platform
sixgill_utils.update_event(
feed_dict,
indicator_list,
indicator,
container,
sixgill_container,
sixgill_artifact,
self._verify_ssl,
)
except:
self._connector.debug_print(
"Error occurred while deleting or updating the event"
)
action_result.set_status(phantom.APP_SUCCESS)
else:
try:
# Ingest the indicator in to the phantom if the indicator does not exist in the phantom SOAR platform
container_status, container_message, container_id = self._connector.save_container(
container)
except:
self._connector.debug_print(
"Error occurred while creating the container")
if container_status == phantom.APP_SUCCESS:
for indicator_dict in indicator_list:
try:
artifacts = sixgill_artifact.prepare_artifact(
container_id, container["severity"],
indicator, indicator_dict)
artifact_status, artifact_message, artifact_id_list = self._connector.save_artifacts(
artifacts)
except:
self._connector.debug_print(
"Error occurred while creating the artifact"
)
action_result.set_status(phantom.APP_SUCCESS)
else:
# Return error status if the indicator didn't get ingested
action_result.set_status(phantom.APP_ERROR)
else:
self._connector.save_progress(
"There is no new Threat Intelligence available")
action_result.set_status(phantom.APP_SUCCESS)
self._connector.debug_print(
f"Indicators Failed to ingest log: {failuer_indicator_list}")
self._connector.save_progress("Sixgill Darkfeed Intelligence ingested")
try:
# Indicators which got ingested into the Phantom SOAR platform will get commited to the Sixgill Darkfeed API end point
darkfeed_client.commit_indicators()
except Exception as e:
err = self._connector._get_error_message_from_exception(e)
return action_result.set_status(
phantom.APP_ERROR,
"Error occurred while committing the indicator(s). {}".format(
err))
return action_result.get_status()