def __init__(self, tcex, main_type, api_type, sub_type, api_entity,
api_branch, owner):
"""Initialize Class Properties.
Args:
tcex:
main_type:
api_type:
sub_type:
api_entity:
"""
self._tcex = tcex
self._data = {}
self._owner = owner
self._type = main_type
self._api_sub_type = sub_type
self._api_type = api_type
self._unique_id = None
self._api_entity = api_entity
self._api_branch = api_branch
self._utils = Utils()
self._tc_requests = TiTcRequest(self._tcex)
class TestCase:
"""Base TestCase Class"""
_app_path = os.getcwd()
_current_test = None
_default_args = None
_profile = None
_session_admin = None
_session_exchange = None
_stager = None
_staged_tc_data = []
_timer_class_start = None
_timer_method_start = None
_validator = None
app = None
context = None
enable_update_profile = False
env = set(os.getenv('TCEX_TEST_ENVS', 'build').split(','))
env_store = EnvStore(logger=logger)
ij = InstallJson(logger=logger)
initialized = False
log = logger
redis_client = None
session = Session()
tcex = None
tcex_testing_context = None
use_token = True
_skip = False
utils = Utils()
def _reset_property_flags(self):
"""Reset all control flag."""
# used to prevent pytest from executing @property methods
self.initialized = False
self.tcex = False
def _log_args(self, args):
"""Log args masking any that are marked encrypted and log warning for unknown args.
Args:
args (dict): A dictionary of args.
"""
for name, value in sorted(args.items()):
input_data = self.ij.params_dict.get(name)
if input_data is None and name not in self.default_args:
# if name is not in install.json and not a default arg log a warning
if name not in ['tcex_testing_context'
]: # values that should not cause a warning
self.log.data('run', 'input', f'{name}: unknown arg.',
'warning')
elif input_data is not None and input_data.get('encrypt') is True:
self.log.data('run', 'input', f'{name}: ***')
else:
self.log.data('run', 'input', f'{name}: {value}')
@staticmethod
def _to_bool(value):
"""Return bool value from int or string."""
return str(value).lower() in ['1', 'true']
def _update_args(self, args: dict) -> dict:
"""Update args before running App.
Args:
args: The current argument dictionary.
Returns:
dict: The updated argument dictionary.
"""
if self.ij.runtime_level.lower() in ['playbook']:
# set requested output variables
args[
'tc_playbook_out_variables'] = self.profile.tc_playbook_out_variables
# update path args
self._update_path_args(args)
# update default args with app args
app_args = self.default_args.copy()
app_args.update(args)
# update default args with app args
app_args['tc_logger_name'] = self.context
# safely log all args to tests.log
self._log_args(app_args)
return app_args
def _update_path_args(self, args):
"""Update path in args for each test profile."""
# service Apps do not have a profile when this is needed.
profile = self.profile or Profile(
default_args=self.default_args.copy())
args['tc_in_path'] = profile.tc_in_path
args['tc_log_path'] = profile.tc_log_path
args['tc_out_path'] = profile.tc_out_path
args['tc_temp_path'] = profile.tc_temp_path
@property
def api_access_id(self):
"""Return TC API Access ID"""
return self.env_store.getenv(
'/ninja/tc/tci/exchange_admin/api_access_id')
@property
def api_secret_key(self):
"""Return TC API Secret Key"""
return self.env_store.getenv(
'/ninja/tc/tci/exchange_admin/api_secret_key')
def check_environment(self, environments, os_environments=None):
"""Check if test case matches current environments, else skip test.
Args:
environments (list): The test case environments.
os_environments (list): The user/os defined environment.
"""
test_envs = environments or ['build']
os_envs = set(os.environ.get('TCEX_TEST_ENVS', 'build').split(','))
if os_environments:
os_envs = set(os_environments)
# APP-1212 - fix issue where outputs were being deleted when profile was skipped
self._skip = False
if not os_envs.intersection(set(test_envs)):
self._skip = True
pytest.skip('Profile skipped based on current environment.')
def create_config(self, args):
"""Create files necessary to start a Service App."""
config = self._update_args(args)
# service Apps will get their args/params from encrypted file in the "in" directory
data = json.dumps(config, sort_keys=True).encode('utf-8')
key = ''.join(random.choice(string.ascii_lowercase) for i in range(16))
encrypted_data = self.utils.encrypt_aes_cbc(key, data)
# ensure that the in directory exists
os.makedirs(config.get('tc_in_path'), exist_ok=True)
# write the file in/.app_params.json
app_params_json = os.path.join(config.get('tc_in_path'),
'.test_app_params.json')
with open(app_params_json, 'wb') as fh:
fh.write(encrypted_data)
# create environment variable for tcex inputs method to pick up to read encrypted file
os.environ['TC_APP_PARAM_KEY'] = key
os.environ['TC_APP_PARAM_FILE'] = app_params_json
@property
def default_args(self):
"""Return App default args."""
if self._default_args is None and self.initialized:
self._default_args = {
# local override TCI_EXCHANGE_ADMIN_API_ACCESS_ID
'api_access_id':
self.api_access_id,
'api_default_org':
os.getenv('API_DEFAULT_ORG', 'TCI'),
# local override TCI_EXCHANGE_ADMIN_API_SECRET_KEY
'api_secret_key':
self.api_secret_key,
'tc_api_path':
self.tc_api_path,
'tc_in_path':
os.getenv('TC_IN_PATH', 'log'),
'tc_log_level':
os.getenv('TC_LOG_LEVEL', 'trace'),
'tc_log_path':
os.getenv('TC_LOG_PATH', 'log'),
'tc_log_to_api':
self._to_bool(os.getenv('TC_LOG_TO_API', 'false')),
'tc_logger_name':
'tcex',
'tc_out_path':
os.getenv('TC_OUT_PATH', 'log'),
'tc_proxy_external':
self._to_bool(os.getenv('TC_PROXY_EXTERNAL', 'false')),
# local override TC_PROXY_HOST
'tc_proxy_host':
self.env_store.getenv('/ninja/proxy/tc_proxy_host',
default='localhost'),
# local override TC_PROXY_PASSWORD
'tc_proxy_password':
self.env_store.getenv('/ninja/proxy/tc_proxy_password',
default=''),
# local override TC_PROXY_PORT
'tc_proxy_port':
self.env_store.getenv('/ninja/proxy/tc_proxy_port',
default='4242'),
'tc_proxy_tc':
self._to_bool(os.getenv('TC_PROXY_TC', 'false')),
# local override TC_PROXY_USERNAME
'tc_proxy_username':
self.env_store.getenv('/ninja/proxy/tc_proxy_username',
default=''),
'tc_temp_path':
os.getenv('TC_TEMP_PATH', 'log'),
}
# try to use token when possible
if self.use_token is True:
token = os.getenv('TC_TOKEN', self.tc_token)
if token is not None:
# if token was successfully retrieved from TC use token and remove hmac values
self._default_args['tc_token'] = token
self._default_args['tc_token_expires'] = int(
time.time()) + 3600
del self._default_args['api_access_id']
del self._default_args['api_secret_key']
return self._default_args
def init_profile(self,
profile_name,
pytestconfig=None,
monkeypatch=None,
options=None):
"""Stages and sets up the profile given a profile name"""
self._profile = Profile(
default_args=self.default_args.copy(),
name=profile_name,
pytestconfig=pytestconfig,
monkeypatch=monkeypatch,
redis_client=self.redis_client,
tcex_testing_context=self.tcex_testing_context,
logger=self.log,
options=options,
)
# Override test environments if specified
os_environments = None
if pytestconfig:
os_environments = pytestconfig.option.environment
# check profile environment
self.check_environment(self._profile.environments, os_environments)
# migrate profile to latest schema
self._profile.migrate()
# merge profile inputs (add new / remove non-defined)
self._profile.merge_inputs()
# populate profile (env vars, etc)
self._profile.data = self._profile.populate.replace_env_variables(
self._profile.data)
# validate input fields, this method requires env vars to be populated
valid, message = self._profile.validate_inputs()
# stage ThreatConnect data based on current profile, also used in teardown method
self._staged_tc_data = self.stager.threatconnect.entities(
self._profile.stage_threatconnect, self._profile.owner)
# insert staged data for replacement
self._profile.tc_staged_data = self._staged_tc_data
# Replace staged_data
self._profile.data = self._profile.populate.replace_tc_variables(
self._profile.data)
# replace all references and all staged variable
self._profile.init()
# stage kvstore data based on current profile
self.stager.redis.from_dict(self._profile.stage_kvstore)
return valid, message
@property
def profile(self):
"""Return profile instance."""
return self._profile
def run(self):
"""Implement in Child Class"""
raise NotImplementedError('Child class must implement this method.')
def run_app_method(self, app, method):
"""Run the provided App method."""
try:
getattr(app, method)()
except SystemExit as e:
self.log.data('run', 'Exit Code', e.code)
if e.code != 0 and self.profile and e.code not in self.profile.exit_codes:
self.log.data(
'run',
'App failed',
f'App exited with code of {e.code} in method {method}',
'error',
)
app.tcex.log.info(f'Exit Code: {e.code}')
return e.code
except Exception:
self.log.data(
'run',
'App failed',
f'App encountered except in {method}() method ({traceback.format_exc()})',
'error',
)
return 1
return 0
@property
def session_admin(self):
"""Return requests Session object for TC admin account.
The credential this session uses require special activation in the ThreatConnect Platform
and is not intended for normal use.
"""
# APP-102 - adding a session for TC API Admin role request
api_access_id = self.env_store.getenv(
'/ninja/tc/system/admin_api/api_access_id')
api_secret_key = self.env_store.getenv(
'/ninja/tc/system/admin_api/api_secret_key')
if (self._session_admin is None and api_access_id is not None
and api_secret_key is not None
and self.tc_api_path is not None):
# support for a proxy is not a typical use case, but can be added later if needed
self._session_admin = TestSession(
api_access_id=api_access_id,
api_secret_key=api_secret_key,
api_path=self.tc_api_path,
logger=self.log,
)
return self._session_admin
@property
def session_exchange(self):
"""Return requests Session object for TC admin account.
The credential this session uses require special activation in the ThreatConnect Platform
and is not intended for normal use.
"""
# APP-102 - adding a session for TC Exchange Admin role request
if (self._session_exchange is None and self.api_access_id is not None
and self.api_secret_key is not None
and self.tc_api_path is not None):
# support for a proxy is not a typical use case, but can be added later if needed
self._session_exchange = TestSession(
api_access_id=self.api_access_id,
api_secret_key=self.api_secret_key,
api_path=self.tc_api_path,
logger=self.log,
)
return self._session_exchange
@classmethod
def setup_class(cls):
"""Run once before all test cases."""
cls.initialized = True
cls._timer_class_start = time.time()
cls.log.title('Setup Class', '#')
TestCase.log.data('setup class', 'started', datetime.now().isoformat())
TestCase.log.data('setup class', 'local envs', cls.env)
def setup_method(self):
"""Run before each test method runs."""
self._timer_method_start = time.time()
self._current_test = os.getenv('PYTEST_CURRENT_TEST').split(' ')[0]
self.log.title(self._current_test, '=')
self.log.data('setup method', 'started', datetime.now().isoformat())
# create and log current context
self.context = os.getenv('TC_PLAYBOOK_DB_CONTEXT', str(uuid.uuid4()))
self.log.data('setup method', 'context', self.context)
# setup per method instance of tcex
args = self.default_args.copy()
args['tc_log_file'] = os.path.join(self.test_case_log_test_dir,
'setup.log')
args[
'tc_logger_name'] = f'tcex-{self.test_case_feature}-{self.test_case_name}'
self.tcex = TcEx(config=args)
# initialize new stager instance
self._stager = self.stager_init()
# initialize new validator instance
self._validator = self.validator_init()
# Adding this for batch to created the -batch and errors files
os.makedirs(os.path.join(self.test_case_log_test_dir, 'DEBUG'),
exist_ok=True)
@property
def stager(self):
"""Return instance of Stager class."""
return self._stager
def stager_init(self):
"""Return instance of Stager class."""
tc_log_file = os.path.join(self.test_case_log_test_dir, 'stage.log')
# args data
args = self.default_args.copy()
# override default log level if profiled
args['tc_log_level'] = 'warning'
# set log path to be the feature and test case name
args['tc_log_file'] = tc_log_file
# set a logger name to have a logger specific for stager
args['tc_logger_name'] = 'tcex-stager'
tcex = TcEx(config=args)
return Stager(tcex, logger)
@property
def tc_api_path(self):
"""Return TC Api Path."""
return os.getenv('TC_API_PATH')
@property
def tc_token(self):
"""Return a valid API token.
note:: requires TC >= 6.0
"""
if self.tc_api_path is None: # no API path, no token
return None
data = None
token = None
token_url_path = self.env_store.getenv('/ninja/tc/token/url_path',
env_type='remote')
if token_url_path is None:
# could not retrieve URL path
return None
# determine the token type
token_type = 'api'
# per conversation with Marut, we should be able to just use api tokens
# if self.ij.runtime_level.lower() in [
# 'apiservice',
# 'triggerservice',
# 'webhooktriggerservice',
# ]:
# data = {'serviceId': os.getenv('TC_TOKEN_SVC_ID', '441')}
# token_type = 'svc'
# retrieve token from API using HMAC auth
r = self.session_exchange.post(f'{token_url_path}/{token_type}',
json=data,
verify=True)
if r.status_code == 200:
token = r.json().get('data')
self.log.data('setup', 'Using Token', token)
self.log.data('setup', 'Token Elapsed', r.elapsed, 'trace')
else:
self.log.error(f'Failed to retrieve token ({r.text})')
return token
@classmethod
def teardown_class(cls):
"""Run once before all test cases."""
cls.initialized = False
cls.log.title('Teardown Class', '^')
TestCase.log.data('teardown class', 'finished',
datetime.now().isoformat())
TestCase.log.data('teardown class', 'elapsed',
time.time() - cls._timer_class_start)
def teardown_method(self):
"""Run after each test method runs."""
if (self.enable_update_profile
and self.ij.runtime_level.lower() not in [
'apiservice', 'triggerservice', 'webhooktriggerservice'
] and self._skip is False):
# exit message can not be validated for a Service App
self.profile.update_exit_message()
# delete threatconnect staged data
self.stager.threatconnect.delete_staged(self._staged_tc_data)
# log running times
self.log.data('teardown method', 'finished',
datetime.now().isoformat())
self.log.data('teardown method', 'elapsed',
time.time() - self._timer_class_start)
# APP-262 - close out loggers
self.tcex.logger.shutdown()
self.stager.tcex.logger.shutdown()
self.validator.tcex.logger.shutdown()
# APP-262 - disconnect redis
self.tcex.redis_client.connection_pool.disconnect()
self.stager.tcex.redis_client.connection_pool.disconnect()
self.validator.tcex.redis_client.connection_pool.disconnect()
# update profile for session data
if self.profile: # doesn't exist for API services
self.profile.session_manager.update_profile()
@property
def test_case_data(self):
"""Return partially parsed test case data."""
return os.getenv('PYTEST_CURRENT_TEST').split(' ')[0].split('::')
@property
def test_case_feature(self):
"""Return partially parsed test case data."""
return self.test_case_data[0].split('/')[1].replace('/', '-')
@property
def test_case_feature_dir(self):
"""Return profile fully qualified filename."""
return os.path.join(self._app_path, 'tests', self.test_case_feature)
@property
def test_case_log_feature_dir(self):
"""Return profile fully qualified filename."""
return os.path.join(self._app_path, os.getenv('TC_LOG_PATH', 'log'),
self.test_case_feature)
@property
def test_case_log_test_dir(self):
"""Return profile fully qualified filename."""
return os.path.join(self.test_case_log_feature_dir,
self.test_case_name)
@property
def test_case_name(self):
"""Return partially parsed test case data."""
return self.test_case_data[-1].replace('/', '-').replace('[',
'-').replace(
']', '')
def validate_exit_message(self, test_exit_message, op='eq', **kwargs):
"""Validate App exit message."""
if test_exit_message is not None:
message_tc_file = os.path.join(
os.getenv('TC_OUT_PATH', 'log'),
self.test_case_feature,
self.test_case_name,
'message.tc',
)
app_exit_message = None
if os.path.isfile(message_tc_file):
with open(message_tc_file) as mh:
app_exit_message = mh.read()
if app_exit_message:
kwargs['title'] = 'Exit Message Validation'
kwargs['log_app_data'] = json.dumps(app_exit_message)
if op == 'eq':
kwargs['log_test_data'] = json.dumps(test_exit_message)
# compare
passed, assert_error = self.validator.compare(
app_exit_message, test_exit_message, op=op, **kwargs)
assert passed, assert_error
else:
assert False, 'The message.tc file was empty.'
else:
assert False, f'No message.tc file found at ({message_tc_file}).'
@property
def validator(self):
"""Return instance of Stager class."""
return self._validator
def validator_init(self):
"""Return instance of Stager class."""
tc_log_file = os.path.join(self.test_case_log_test_dir, 'validate.log')
# args data
args = self.default_args.copy()
# override default log level if profiled
args['tc_log_level'] = 'warning'
# set log path to be the feature and test case name
args['tc_log_file'] = tc_log_file
# set a logger name to have a logger specific for stager
args['tc_logger_name'] = 'tcex-validator'
tcex = TcEx(config=args)
return Validator(tcex, logger)
class MockApp:
"""MockApp Class for testing TcEx Framework.
Typical usage when using conftest.py fixture:
# basic option
tcex = service_app().tcex
# with runlevel option
tcex = service_app(runtime_level='WebhookTriggerService').tcex
Args:
runtime_level (str): The runtime level of the Mock App.
clear_argv (bool, optional): [description]. Defaults to True.
config_data ([type], optional): [description]. Defaults to None.
ij_data ([type], optional): [description]. Defaults to None.
"""
def __init__(self, runtime_level, **kwargs):
"""Initialize class properties."""
self.runtime_level = runtime_level
self.cd = kwargs.get('config_data',
{}) # configuration data for tcex instance
self.clear_argv = kwargs.get('clear_argv', True) # clear sys.argv
self.ijd = kwargs.get('ij_data', {}) # install.json data
# properties
api_access_id = os.getenv('API_ACCESS_ID')
api_secret_key = os.getenv('API_SECRET_KEY')
self._config_data = None
self.tc_api_path = os.getenv('TC_API_PATH')
self.tc_token_url = os.getenv('TC_TOKEN_URL')
self.tc_token_svc_id = os.getenv('TC_TOKEN_SVC_ID')
# get a requests session and set hmac auth to use in retrieving tokens.
self.session = Session()
self.session.auth = HmacAuth(api_access_id, api_secret_key)
# utils method
self.utils = Utils()
# create install.json file
self._create_install_json()
def _build_config_data(self):
"""Return config data for mocked App."""
config = {
# default
'api_default_org': self.getenv('api_default_org'),
'tc_owner': self.getenv('tc_owner', 'TCI'),
# logging
'tc_log_level': self.getenv('tc_log_level', 'trace'),
'tc_log_to_api': self.getenv('tc_log_to_api', 'false', True),
# paths
'tc_api_path': self.getenv('tc_api_path'),
'tc_in_path': self.getenv('tc_in_path', 'log'),
'tc_log_path': self.getenv('tc_log_path', 'log'),
'tc_out_path': self.getenv('tc_out_api', 'log'),
'tc_temp_path': self.getenv('tc_temp_path', 'log'),
# proxy
'tc_proxy_tc': self.getenv('tc_proxy_tc', 'false', True),
'tc_proxy_external': self.getenv('tc_proxy_external', 'false',
True),
}
# add specific config shared between job and playbook Apps
if self.runtime_level.lower() in ['job', 'playbook']:
# 'tc_token': self.service_token,
config['tc_token'] = self.api_token
config['tc_token_expires'] = '1700000000'
# hmac auth (for session tests)
config['api_access_id'] = self.getenv('api_access_id')
config['api_secret_key'] = self.getenv('api_secret_key')
# add specific config options for playbook and service Apps
if self.runtime_level.lower() in [
'apiservice',
'playbook',
'triggerservice',
'webhooktriggerservice',
]:
config['tc_playbook_db_type'] = self.getenv(
'tc_playbook_db_type', 'Redis')
config['tc_playbook_db_context'] = self.getenv(
'tc_playbook_db_context', str(uuid.uuid4()))
config['tc_playbook_db_path'] = self.getenv(
'tc_playbook_db_path', 'localhost')
config['tc_playbook_db_port'] = self.getenv(
'tc_playbook_db_port', '6379')
# add specific config options for service Apps
if self.runtime_level.lower() in [
'apiservice',
'triggerservice',
'webhooktriggerservice',
]:
config['tc_svc_client_topic'] = self.getenv(
'tc_playbook_db_port',
'svc-client-cc66d36344787779ccaa8dbb5e09a7ab')
# add proxy config options if appropriate
if self.getenv('tc_proxy_host'):
config['tc_proxy_host'] = self.getenv('tc_proxy_host')
if self.getenv('tc_proxy_port'):
config['tc_proxy_port'] = self.getenv('tc_proxy_port')
if self.getenv('tc_proxy_username'):
config['tc_proxy_username'] = self.getenv('tc_proxy_username')
if self.getenv('tc_proxy_password'):
config['tc_proxy_password'] = self.getenv('tc_proxy_password')
# create log structure for feature/test (e.g., args/test_args.log)
config['tc_log_file'] = self.tcex_log_file
# anything remaning in self.cd would be an arg to add.
for k, v in self.cd.items():
config[k] = v
return config
def _create_app_params_file(self, config):
"""Create the app param file for service Apps."""
config_data = json.dumps(config).encode()
config_key = self.utils.random_string(16)
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'app_params.aes')
# encrypt the serialized config data
encrypted_contents = self.utils.encrypt_aes_cbc(
config_key, config_data)
# write the config data to disk
with open(config_file, 'wb') as fh:
fh.write(encrypted_contents)
# set the environment variables for tcex input module.
os.environ['TC_APP_PARAM_FILE'] = config_file
os.environ['TC_APP_PARAM_KEY'] = config_key
def _create_install_json(self):
"""Create a mock install.json file."""
with open('install.json', 'w') as fh:
json.dump(self._mock_install_json, fh, indent=2)
@property
def _mock_install_json(self):
"""Return install.json data for mocked App."""
display_name = self.ijd.get('display_name', 'Pytest')
features = self.ijd.get('features') or [
'aotExecutionEnabled',
'appBuilderCompliant',
'layoutEnabledApp',
'secureParams',
]
params = self.ijd.get('params') or [
{
'label': 'My Bool',
'name': 'my_bool',
'note': '',
'required': True,
'sequence': 1,
'type': 'Boolean',
},
{
'label': 'My Multi',
'name': 'my_multi',
'note': '',
'required': False,
'sequence': 8,
'type': 'MultiChoice',
'validValues': ['one', 'two'],
},
]
playbook = self.ijd.get('playbook') or {
'outputVariables': [],
'type': 'Utility'
}
ij = {
'allowOnDemand': True,
'commitHash': 'abc-123',
'displayName': display_name,
'features': features,
'languageVersion': '3.6',
'listDelimiter': '|',
'note': '',
'params': params,
'programLanguage': 'PYTHON',
'programMain': 'run',
'programVersion': '1.0.0',
'runtimeLevel': self.runtime_level,
}
if self.runtime_level.lower() != 'job':
ij['playbook'] = playbook
return ij
@property
def api_token(self):
"""Return a valid API token."""
r = self.session.post(f'{self.tc_api_path}{self.tc_token_url}api',
verify=False)
if r.status_code != 200:
raise RuntimeError(
f'This feature requires ThreatConnect 6.0 or higher ({r.text})'
)
return r.json().get('data')
@property
def config_data(self):
"""Return the configuration data for that App or write to app_param."""
if self._config_data is None:
self._config_data = self._build_config_data()
if self.runtime_level.lower() in [
'apiservice',
'triggerservice',
'webhooktriggerservice',
]:
# add the config to the encrypted file and send TC empty dict
self._create_app_params_file(self._config_data)
self._config_data = {}
return self._config_data
def getenv(self, key, default=None, boolean=False):
"""Get the appropriate **config value**.
Use config_data value provided to Class,
else use environment variable data,
else use default value
Args:
key (str): The key value.
default (str, optional): The default value to return if not found. Defaults to None.
boolean (bool, optional): If true return the result as a bool. Defaults to False.
Returns:
bool|str: The value found in config date, env var, or default.
"""
cv = os.getenv(key.upper(), default)
if hasattr(self.cd, key):
# check if it exist so None can be set
cv = self.cd.pop(key)
if boolean:
# convert string response to boolean
cv = str(cv).lower() in ['true']
return cv
@property
def service_token(self):
"""Get a valid TC service token.
TC_TOKEN_SVC_ID is the ID field from the appcatalogitem table for a service App.
TC_TOKEN_URL is the API endpoint to get a TOKEN.
"""
data = {'serviceId': os.getenv('TC_TOKEN_SVC_ID')}
r = self.session.post(f'{self.tc_api_path}{self.tc_token_url}svc',
json=data,
verify=False)
if r.status_code != 200:
raise RuntimeError(
f'This feature requires ThreatConnect 6.0 or higher ({r.text})'
)
return r.json().get('data')
@property
def tcex(self):
"""Return an instance of tcex."""
# clear sys.argv to avoid invalid arguments
if self.clear_argv:
sys.argv = sys.argv[:1]
tcex = TcEx(config=self.config_data)
# cleanup environment variables
if os.getenv('TC_APP_PARAM_FILE', None):
del os.environ['TC_APP_PARAM_FILE']
if os.getenv('TC_APP_PARAM_KEY', None):
del os.environ['TC_APP_PARAM_KEY']
return tcex
@property
def tcex_log_file(self):
"""Return log file name for current test case."""
try:
test_data = os.getenv('PYTEST_CURRENT_TEST').split(' ')[0].split(
'::')
test_feature = test_data[0].split('/')[1].replace('/', '-')
test_name = test_data[-1].replace('/', '-').replace('[', '-')
except AttributeError:
# TODO: remove this once tcex_init file is removed
test_feature = 'tcex_init_legacy'
test_name = 'app'
return os.path.join(test_feature, f'{test_name}.log')
class TestSession(Session):
"""ThreatConnect REST API Requests Session
Args:
api_access_id (str): The ThreatConnect API access id.
api_secret_key (str): The ThreatConnect API secret key.
proxy_host (str, kwargs): The proxy hostname.
proxy_port (int, kwargs): The proxy port number.
proxy_user (str, kwargs): The proxy username.
proxy_pass (str, kwargs): The proxy password.
verify (bool, kwargs): If True SSL verification of the host will be performed.
"""
def __init__(self, api_access_id, api_secret_key, api_path, logger,
**kwargs):
"""Initialize the Class properties."""
super().__init__()
self.api_path = api_path
self.api_access_id = api_access_id
self.api_secret_key = api_secret_key
self.log = logger
self.proxy_host = kwargs.get('proxy_host')
self.proxy_port = kwargs.get('proxy_port')
self.proxy_user = kwargs.get('proxy_user')
self.proxy_pass = kwargs.get('proxy_pass')
# properties
self.auth = None
self.utils = Utils()
# Update User-Agent
self.headers.update({'User-Agent': 'TcEx Testing'})
# Set Proxy
self.proxies = self.proxy_config
# Add Retry
self.retry()
# Set Verify
self.verify = kwargs.get('verify', True)
def _configure_auth(self):
"""Return Auth property for session."""
if self.api_access_id and self.api_secret_key:
try:
# for external Apps or testing Apps locally
self.auth = HmacAuth(self.api_access_id, self.api_secret_key)
self.log.debug('Using HMAC authorization.')
except AttributeError: # pragma: no cover
raise RuntimeError(
'No valid ThreatConnect API credentials provided.')
else: # pragma: no cover
raise RuntimeError(
'No valid ThreatConnect API credentials provided.')
@property
def proxy_config(self):
"""Format the proxy configuration for Python Requests module.
Generates a dictionary for use with the Python Requests module format
when proxy is required for remote connections.
**Example Response**
::
{"http": "http://*****:*****@10.10.1.10:3128/"}
Returns:
(dictionary): Dictionary of proxy settings
"""
proxies = {}
if self.proxy_host is not None and self.proxy_port is not None:
if self.proxy_username is not None and self.proxy_password is not None:
proxy_username = quote(self.proxy_username, safe='~')
proxy_password = quote(self.proxy_password, safe='~')
# proxy url with auth
proxy_url = (f'{proxy_username}:{proxy_password}'
f'@{self.proxy_host}:{self.proxy_port}')
else:
# proxy url without auth
proxy_url = f'{self.proxy_host}:{self.proxy_port}'
proxies = {
'http': f'http://{proxy_url}',
'https': f'https://{proxy_url}'
}
return proxies
def request(self, method, url, **kwargs): # pylint: disable=arguments-differ
"""Override request method disabling verify on token renewal if disabled on session."""
if self.auth is None:
self._configure_auth()
if not url.startswith('https'):
# automatically add api path to each request
url = f'{self.api_path}{url}'
response = super().request(method, url, **kwargs)
# APP-79 - adding logging of request as curl commands
self.log.debug(
self.utils.requests_to_curl(response.request, verify=self.verify))
return response
def retry(self,
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504)):
"""Add retry to Requests Session
https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
"""
retries = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
# mount all https requests
self.mount('https://', adapters.HTTPAdapter(max_retries=retries))
class ThreatIntelligence:
"""ThreatConnect Threat Intelligence Module"""
def __init__(self, session: Session) -> None:
"""Initialize Class properties."""
self.session = session
# properties
self._custom_indicator_classes = {}
self.log = logger
self.utils = Utils()
# generate custom ioc classes
self._gen_indicator_class()
@property
@lru_cache()
def _error_codes(self) -> TcExErrorCodes: # noqa: F821
"""Return TcEx error codes."""
return TcExErrorCodes()
@property
def _group_types(self) -> list:
"""Return all defined ThreatConnect Group types.
Returns:
(list): A list of ThreatConnect Group types.
"""
return [
'Adversary',
'Campaign',
'Document',
'Email',
'Event',
'Incident',
'Intrusion Set',
'Signature',
'Report',
'Threat',
'Task',
'Attack Pattern',
'Malware',
'Vulnerability',
'Tactic',
'Tool',
'Course of Action',
]
@property
def _group_types_data(self) -> dict:
"""Return supported ThreatConnect Group types."""
return {
'Adversary': {
'apiBranch': 'adversaries',
'apiEntity': 'adversary'
},
'Campaign': {
'apiBranch': 'campaigns',
'apiEntity': 'campaign'
},
'Document': {
'apiBranch': 'documents',
'apiEntity': 'document'
},
'Email': {
'apiBranch': 'emails',
'apiEntity': 'email'
},
'Event': {
'apiBranch': 'events',
'apiEntity': 'event'
},
'Incident': {
'apiBranch': 'incidents',
'apiEntity': 'incident'
},
'Intrusion Set': {
'apiBranch': 'intrusionSets',
'apiEntity': 'intrusionSet'
},
'Report': {
'apiBranch': 'reports',
'apiEntity': 'report'
},
'Signature': {
'apiBranch': 'signatures',
'apiEntity': 'signature'
},
'Threat': {
'apiBranch': 'threats',
'apiEntity': 'threat'
},
'Task': {
'apiBranch': 'tasks',
'apiEntity': 'task'
},
'Attack Pattern': {
'apiBranch': 'attackpatterns',
'apiEntity': 'attackPattern'
},
'Malware': {
'apiBranch': 'malware',
'apiEntity': 'malware'
},
'Vulnerability': {
'apiBranch': 'vulnerabilities',
'apiEntity': 'vulnerability'
},
'Tactic': {
'apiBranch': 'tactics',
'apiEntity': 'tactic'
},
'Tool': {
'apiBranch': 'tools',
'apiEntity': 'tool'
},
'Course of Action': {
'apiBranch': 'coursesofaction',
'apiEntity': 'courseofAction'
},
}
@property
@lru_cache()
def _indicator_types_data(self) -> dict:
"""Return ThreatConnect indicator types data.
Retrieve the data from the API if it hasn't already been retrieved.
Returns:
(dict): A dictionary of ThreatConnect Indicator data.
"""
_indicator_types_data = {}
# retrieve data from API
r = self.session.get('/v2/types/indicatorTypes')
# TODO: use handle error instead
if not r.ok:
raise RuntimeError(
'Could not retrieve indicator types from ThreatConnect API.')
for itd in r.json().get('data', {}).get('indicatorType'):
_indicator_types_data[itd.get('name')] = itd
return _indicator_types_data
def _handle_error(self,
code: int,
message_values: Optional[list] = None,
raise_error: Optional[bool] = True) -> None:
"""Raise RuntimeError
Args:
code: The error code from API or SDK.
message: The error message from API or SDK.
raise_error: Raise a Runtime error. Defaults to True.
Raises:
RuntimeError: Raised a defined error.
"""
try:
if message_values is None:
message_values = []
message = self._error_codes.message(code).format(*message_values)
self.log.error(f'Error code: {code}, {message}')
except AttributeError:
self.log.error(f'Incorrect error code provided ({code}).')
raise RuntimeError(100,
'Generic Failure, see logs for more details.')
except IndexError:
self.log.error(
f'Incorrect message values provided for error code {code} ({message_values}).'
)
raise RuntimeError(100,
'Generic Failure, see logs for more details.')
if raise_error:
raise RuntimeError(code, message)
def address(self, **kwargs):
"""Return an Address TI object.
Args:
ip (str, kwargs): [Required for Create] The IP value for this Indicator.
active (bool, kwargs): If False the indicator is marked "inactive" in TC.
confidence (str, kwargs): The threat confidence for this Indicator.
date_added (str, kwargs): [Read-Only] The date timestamp the Indicator was created.
last_modified (str, kwargs): [Read-Only] The date timestamp the Indicator was last
modified.
private_flag (bool, kwargs): If True the indicator is marked as private in TC.
rating (str, kwargs): The threat rating for this Indicator.
Returns:
obj: An instance of Address.
"""
return Address(self, **kwargs)
def url(self, **kwargs):
"""Create the URL TI object.
Args:
owner (str, kwargs): The name for this Group. Default to default Org when not provided
text (str, kwargs): [Required for Create] The URL value for this Indicator.
Return:
obj: An instance of URL.
"""
return URL(self, **kwargs)
def email_address(self, **kwargs):
"""Create the Email Address TI object.
Args:
owner (str, kwargs): The name for this Group. Default to default Org when not provided
address (str, kwargs): [Required for Create] The Email Address value for this Indicator.
Return:
obj: An instance of EmailAddress.
"""
return EmailAddress(self, **kwargs)
def file(self, **kwargs):
"""Create the File TI object.
Args:
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
obj: An instance of File.
"""
return File(self, **kwargs)
def host(self, **kwargs):
"""Create the Host TI object.
Args:
owner (str, kwargs): The name for this Group. Default to default Org when not provided
hostname (str, kwargs): [Required for Create] The Host value for this Indicator.
Return:
obj: An instance of Host.
"""
return Host(self, **kwargs)
@staticmethod
def filters():
"""Create a Filters TI object"""
return Filters()
def indicator(self, indicator_type=None, owner=None, **kwargs):
"""Return an TI object.
Args:
active (bool, kwargs): If False the indicator is marked "inactive" in TC.
confidence (str, kwargs): The threat confidence for this Indicator.
date_added (str, kwargs): [Read-Only] The date timestamp the Indicator was created.
indicator_type (str): The indicator type.
ip (str, kwargs): [address] The value for this Indicator.
last_modified (str, kwargs): [Read-Only] The date timestamp the Indicator was last
modified.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
private_flag (bool, kwargs): If True the indicator is marked as private in TC.
rating (str, kwargs): The threat rating for this Indicator.
Returns:
obj: An instance of Indicator or specific indicator type.
"""
if not indicator_type:
return Indicator(self, owner=owner, **kwargs)
indicator_type_map = {
'address': Address,
'email address': EmailAddress,
'emailaddress': EmailAddress,
'file': File,
'host': Host,
'url': URL,
}
indicator_type = indicator_type.lower()
for custom_type in self._custom_indicator_classes:
for a in dir(module):
if a.lower() == custom_type.replace(' ', ''):
indicator_type_map[custom_type.lower()] = getattr(
module, a)
if indicator_type not in indicator_type_map:
raise RuntimeError(
f'Invalid indicator type "{indicator_type}" provided.')
# update kwargs
kwargs['owner'] = owner
# return correct indicator object
indicator_object = indicator_type_map.get(indicator_type)
return indicator_object(self, **kwargs)
def group(self, group_type=None, owner=None, **kwargs):
"""Create the Group TI object.
Args:
owner (str): The ThreatConnect owner name.
group_type: The type of group object.
"""
if not group_type:
return Group(self, owner=owner, **kwargs)
group_type_map = {
'adversary': Adversary,
'campaign': Campaign,
'document': Document,
'event': Event,
'email': Email,
'incident': Incident,
'intrusion set': IntrusionSet,
'report': Report,
'signature': Signature,
'threat': Threat,
'attack pattern': AttackPattern,
'malware': Malware,
'vulnerability': Vulnerability,
'tactic': Tactic,
'tool': Tool,
'course of action': CourseOfAction,
}
# if "name" is not in kwargs
if kwargs.get('name') is None:
kwargs['name'] = None
group_type = group_type.lower()
if group_type not in group_type_map:
raise RuntimeError(f'Invalid group type "{group_type}" provided.')
# update kwargs
kwargs['owner'] = owner
# return correct group object
group_object = group_type_map.get(group_type)
return group_object(self, **kwargs)
def attack_pattern(self, **kwargs):
"""Create the Attack Pattern TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.AttackPattern: An instance of AttackPattern.
"""
return AttackPattern(self, **kwargs)
def malware(self, **kwargs):
"""Create the Malware TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.Malware: An instance of Malware.
"""
return Malware(self, **kwargs)
def vulnerability(self, **kwargs):
"""Create the Vulnerability TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.Vulnerability: An instance of Vulnerability.
"""
return Vulnerability(self, **kwargs)
def tactic(self, **kwargs):
"""Create the Tactic TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.Tactic: An instance of Tactic.
"""
return Tactic(self, **kwargs)
def tool(self, **kwargs):
"""Create the Tool TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.Tool: An instance of Tool.
"""
return Tool(self, **kwargs)
def course_of_action(self, **kwargs):
"""Create the Course of Action TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.CourseOfAction: An instance of CourseOfAction.
"""
return CourseOfAction(self, **kwargs)
def adversary(self, **kwargs):
"""Create the Adversary TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.Adversary: An instance of Adversary.
"""
return Adversary(self, **kwargs)
def campaign(self, **kwargs):
"""Create the Campaign TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
first_seen (str, kwargs): The first seen datetime expression for this Group.
Return:
ti.Campaign: An instance of Campaign.
"""
return Campaign(self, **kwargs)
def document(self, **kwargs):
"""Create the Document TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
file_name (str, kwargs): The name for the attached file for this Group.
malware (bool, kwargs): If true the file is considered malware.
password (bool, kwargs): If malware is true a password for the zip archive is required.
Return:
ti.Document: An instance of Document.
"""
return Document(self, **kwargs)
def email(self, **kwargs):
"""Create the Email TI object.
Args:
body (str): The body for this Email.
from_addr (str, kwargs): The **from** address for this Email.
header (str): The header for this Email.
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
subject (str): The subject for this Email.
to (str, kwargs): The **to** address for this Email.
Return:
ti.Email: An instance of Email.
"""
return Email(self, **kwargs)
def event(self, **kwargs):
"""Create the Event TI object.
Args:
event_date (str, kwargs): The event "event date" datetime expression for this Group.
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
status (str, kwargs): The status for this Group.
Return:
ti.Event: An instance of Event.
"""
return Event(self, **kwargs)
def incident(self, **kwargs):
"""Create the Incident TI object.
Args:
event_date (str, kwargs): The incident event date expression for this Group.
name (str, kwargs): [Required for Create] The name for this Group.
status (str, kwargs): The status for this Group.
Return:
ti.Incident: An instance of Incident.
"""
return Incident(self, **kwargs)
def intrusion_set(self, **kwargs):
"""Create the Intrustion Set TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
Return:
ti.IntrusionSet: An instance of IntrusionSet.
"""
return IntrusionSet(self, **kwargs)
def report(self, **kwargs):
"""Create the Report TI object.
Args:
owner (str): The ThreatConnect owner name.
name:
**kwargs:
"""
return Report(self, **kwargs)
def signature(self, **kwargs):
"""Create the Signature TI object.
Args:
owner (str): The ThreatConnect owner name.
name (str): The name for this Group.
file_name (str): The name for the attached signature for this Group.
file_type (str): The signature type for this Group.
file_text (str): The signature content for this Group.
**kwargs:
Return:
obj: An instance of Signature.
"""
return Signature(self, **kwargs)
def task(self, **kwargs):
"""Create the Task TI object.
Args:
name (str, kwargs): [Required for Create] The name for this Group.
owner (str, kwargs): The name for this Group. Default to default Org when not provided
status (str, kwargs): Not started, In Progress, Completed, Waiting on Someone, Deferred
due_date (str, kwargs): Converted to %Y-%m-%dT%H:%M:%SZ date format
reminder_date (str, kwargs): Converted to %Y-%m-%dT%H:%M:%SZ date format
escalation_date (str, kwargs): Converted to %Y-%m-%dT%H:%M:%SZ date format
Return:
obj: An instance of Task
"""
return Task(self, **kwargs)
def threat(self, **kwargs):
"""Create the Threat TI object.
Args:
owner (str): The ThreatConnect owner name.
name:
**kwargs:
"""
return Threat(self, **kwargs)
def victim(self, **kwargs):
"""Create the Victim TI object.
Args:
owner (str): The ThreatConnect owner name.
name:
**kwargs:
"""
return Victim(self, **kwargs)
def tags(self):
"""Create the Tag TI object."""
return Tags(self)
def tag(self, name):
"""Create the Tag TI object."""
return Tag(self, name)
def owner(self):
"""Create the Owner object."""
return Owner(self)
def create_entity(self, entity, owner):
"""Given a Entity and a Owner, creates a indicator/group in ThreatConnect"""
attributes = entity.pop('attribute', [])
associations = entity.pop('associations', [])
security_labels = entity.pop('securityLabel', [])
tags = entity.pop('tag', [])
entity_type = entity.pop('type', '').lower()
file_content = None
if entity_type in ['document', 'report']:
file_content = entity.pop('file_content', None) or entity.pop(
'fileContent', None)
try:
ti = self.indicator(entity_type, owner, **entity)
if entity.get('falsePositive'):
ti.add_false_positive()
except Exception:
if entity_type in ['victim']:
ti = self.victim(owner=owner, **entity)
else:
entity['name'] = entity.pop('summary', None)
if entity_type in ['task']:
ti = self.task(owner=owner, **entity)
else:
ti = self.group(entity_type, owner, **entity)
r = ti.create()
if entity_type in ['document', 'report']:
ti.file_content(file_content)
data = {'status_code': r.status_code}
if r.ok:
data.update(r.json().get('data', {}))
data['main_type'] = ti.type
data['sub_type'] = ti.api_sub_type
data['api_type'] = ti.api_sub_type
data['api_entity']: ti.api_entity
data['api_branch']: ti.api_branch
data['owner'] = owner
data['attributes'] = []
data['tags'] = []
data['security_labels'] = []
data['associations'] = []
for attribute in attributes:
r = ti.add_attribute(attribute.get('type'), attribute.get('value'))
attribute_data = {'status_code': r.status_code}
if r.ok:
attribute_data.update(r.json().get('attribute', {}))
data['attributes'].append(attribute_data)
for tag in tags:
r = ti.add_tag(tag)
tag_response = {'status_code': r.status_code}
data['tags'].append(tag_response)
for label in security_labels:
r = ti.add_label(label)
label_response = {'status_code': r.status_code}
data['security_labels'].append(label_response)
for association in associations:
association_target = self.indicator(association.pop('type', None),
association.pop('owner', None),
**association)
if not association_target:
association_target = self.group(association.pop('type', None),
association.pop('owner', None),
**association)
r = ti.add_association(association_target)
association_response = {'status_code': r.status_code}
if r.ok:
association_response.update(r.json().get('association', {}))
data['associations'].append(association_response)
return data
def create_entities(self, entities, owner):
"""Create a indicator/group in TC based on the given entity's
Args:
entities: The entity to create.
owner: The owner of the entity (
"""
responses = []
for entity in entities:
responses.append(self.create_entity(entity, owner))
return responses
def entities(self, tc_data, resource_type):
"""Yield an entity.
Takes both a list of indicators/groups or a individual indicator/group response.
example formats
.. code-block:: javascript
{
"status": "Success",
"data": {
"resultCount": 984240,
"address": [
{
"id": 4222035,
"ownerName": "System",
"dateAdded": "2019-03-28T10:32:05-04:00",
"lastModified": "2019-03-28T11:02:46-04:00",
"rating": 4,
"confidence": 90,
"threatAssessRating": 4,
"threatAssessConfidence": 90,
"webLink": "{host}/auth/indicators/details/address.xhtml?
address=221.123.32.14",
"ip": "221.123.32.14"
},
{
"id": 4221517,
"ownerName": "System",
"dateAdded": "2018-11-05T14:24:54-05:00",
"lastModified": "2019-03-07T12:38:36-05:00",
"threatAssessRating": 0,
"threatAssessConfidence": 0,
"webLink": "{host}/auth/indicators/details/address.xhtml?
address=221.123.32.12",
"ip": "221.123.32.12"
}
]
}
}
or:
.. code-block:: javascript
{
"status": "Success",
"data": {
"address": {
"id": 4222035,
"owner": {
"id": 1,
"name": "System",
"type": "Organization"
},
"dateAdded": "2019-03-28T10:32:05-04:00",
"lastModified": "2019-03-28T11:02:46-04:00",
"rating": 4,
"confidence": 90,
"threatAssessRating": 4,
"threatAssessConfidence": 90,
"webLink": "{host}/auth/indicators/details/address.xhtml?
address=221.123.32.14",
"ip": "221.123.32.14"
}
}
}
Args:
tc_data: TC data to convert to a entity.
resource_type: The type of TC data being provided.
"""
if not isinstance(tc_data, list):
tc_data = [tc_data]
for d in tc_data:
entity = {'id': d.get('id'), 'webLink': d.get('webLink')}
values = []
value = None
keys = d.keys()
if resource_type.lower() in map(str.lower, self._group_types):
# @bpurdy - is this okay?
# r = self.tcex.ti.group(group_type=resource_type, name=d.get('name'))
r = self.group(group_type=resource_type, name=d.get('name'))
value = d.get('name')
elif resource_type.lower() in map(
str.lower, self._indicator_types_data.keys()):
# @bpurdy - is this okay?
# r = self.tcex.ti.indicator(indicator_type=resource_type)
r = self.indicator(indicator_type=resource_type)
r._set_unique_id(d)
value = r.unique_id
elif resource_type.lower() in ['victim']:
r = self.victim(name=d.get('name'))
value = d.get('name')
else:
self._handle_error(
925, ['type', 'entities', 'type', 'type', resource_type])
if 'summary' in d:
values.append(d.get('summary'))
else:
if resource_type.lower() in ['file']:
value = r.build_summary(d.get('md5'), d.get('sha1'),
d.get('sha256'))
values.append(r.fully_decode_uri(value))
entity['value'] = ' : '.join(values)
if r.is_group() or r.is_indicator():
if 'owner' in d:
entity['ownerName'] = d['owner']['name']
else:
entity['ownerName'] = d.get('ownerName')
entity['dateAdded'] = d.get('dateAdded')
if r.is_victim():
entity['ownerName'] = d.get('org')
if r.is_indicator():
entity['confidence'] = d.get('confidence')
entity['rating'] = d.get('rating')
entity['threatAssessConfidence'] = d.get(
'threatAssessConfidence')
entity['threatAssessRating'] = d.get('threatAssessRating')
entity['dateLastModified'] = d.get('lastModified')
if 'whoisActive' in keys:
entity['whoisActive'] = d.get('whoisActive')
if 'dnsActive' in keys:
entity['dnsActive'] = d.get('dnsActive')
if r.is_task():
entity['status'] = d.get('status')
entity['escalated'] = d.get('escalated')
entity['reminded'] = d.get('reminded')
entity['overdue'] = d.get('overdue')
entity['dueDate'] = d.get('dueDate', None)
entity['reminderDate'] = d.get('reminderDate', None)
entity['escalationDate'] = d.get('escalationDate', None)
if d.get('xid'):
entity['xid'] = d.get('xid')
if r.is_group():
if 'xid' in keys:
entity['xid'] = d.get('xid')
if 'firstSeen' in keys:
entity['firstSeen'] = d.get('firstSeen')
if 'fileName' in keys:
entity['fileName'] = d.get('fileName')
if 'fileType' in keys:
entity['fileType'] = d.get('fileType')
if 'fileSize' in keys:
entity['fileSize'] = d.get('fileSize')
if 'eventDate' in keys:
entity['eventDate'] = d.get('eventDate')
if 'status' in keys:
entity['status'] = d.get('status')
if 'to' in keys:
entity['to'] = d.get('to')
if 'from' in keys:
entity['from'] = d.get('from')
if 'subject' in keys:
entity['subject'] = d.get('subject')
if 'score' in keys:
entity['score'] = d.get('score')
if 'header' in keys:
entity['header'] = d.get('header')
if 'body' in keys:
entity['body'] = d.get('body')
if 'publishDate' in keys:
entity['publishDate'] = d.get('publishDate')
if r.api_sub_type.lower() in [
'signature', 'document', 'report'
]:
r.unique_id = d.get('id')
content_response = r.download()
if content_response.ok:
entity['fileContent'] = content_response.text
# get the entity type
if d.get('type') is not None:
entity['type'] = d.get('type')
else:
entity['type'] = resource_type
yield entity
def _gen_indicator_class(self):
"""Generate Custom Indicator Classes."""
for entry in self._indicator_types_data.values():
name = entry.get('name')
class_name = name.replace(' ', '')
# temp fix for API issue where boolean are returned as strings
entry['custom'] = self.utils.to_bool(entry.get('custom'))
if class_name in globals():
# skip Indicator Type if a class already exists
continue
# Custom Indicator can have 3 values. Only add the value if it is set.
value_fields = []
if entry.get('value1Label'):
value_fields.append(entry['value1Label'])
if entry.get('value2Label'):
value_fields.append(entry['value2Label'])
if entry.get('value3Label'):
value_fields.append(entry['value3Label'])
if not value_fields:
continue
# Add Class for each Custom Indicator type to this module
custom_class = custom_indicator_class_factory(
entry.get('name'),
entry.get('apiEntity'),
entry.get('apiBranch'),
Indicator,
value_fields,
)
custom_indicator_data = {
'branch': entry.get('apiBranch'),
'entry': entry.get('apiEntry'),
'value_fields': value_fields,
}
self._custom_indicator_classes[entry.get(
'name').lower()] = custom_indicator_data
setattr(module, class_name, custom_class)
# Add Custom Indicator Method
self._gen_indicator_method(name, custom_class)
def _gen_indicator_method(self, name, custom_class):
"""Dynamically generate custom Indicator methods.
Args:
name (str): The name of the method.
custom_class (object): The class to add.
"""
method_name = name.replace(' ', '_').lower()
ti = self
# Add Method for each Custom Indicator class
def method_1(**kwargs): # pylint: disable=possibly-unused-variable
"""Add Custom Indicator data to Batch object"""
return custom_class(ti, **kwargs)
method = locals()['method_1']
setattr(self, method_name, method)