def run() -> None:
"""Update path and run the App."""
# update the path to ensure the App has access to required modules
app_lib = AppLib()
app_lib.update_path()
# import modules after path has been updated
# third-party
from tcex import TcEx # pylint: disable=import-outside-toplevel
# first-party
from app import App # pylint: disable=import-outside-toplevel
tcex = TcEx()
try:
# load App class
app = App(tcex)
# perform prep/setup operations
app.setup(**{})
# run the App logic
if hasattr(app.args, 'tc_action') and app.args.tc_action is not None:
# if the args NameSpace has the reserved arg of "tc_action", this arg value is used to
# triggers a call to the app.<tc_action>() method. an exact match to the method is
# tried first, followed by a normalization of the tc_action value, and finally an
# attempt is made to find the reserved "tc_action_map" property to map value to method.
tc_action: str = app.args.tc_action
tc_action_formatted: str = tc_action.lower().replace(' ', '_')
tc_action_map = 'tc_action_map' # reserved property name for action to method map
# run action method
if hasattr(app, tc_action):
getattr(app, tc_action)()
elif hasattr(app, tc_action_formatted):
getattr(app, tc_action_formatted)()
elif hasattr(app, tc_action_map):
app.tc_action_map.get(app.args.tc_action)() # pylint: disable=no-member
else:
tcex.exit(
1, f'Action method ({app.args.tc_action}) was not found.')
else:
# default to run method
app.run(**{})
# write requested value for downstream Apps
app.write_output() # pylint: disable=no-member
# perform cleanup/teardown operations
app.teardown(**{})
# explicitly call the exit method
tcex.playbook.exit(msg=app.exit_message)
except Exception as e:
main_err = f'Generic Error. See logs for more details ({e}).'
tcex.log.error(traceback.format_exc())
tcex.playbook.exit(1, main_err)
def run() -> None:
"""Update path and run the App."""
# update the path to ensure the App has access to required modules
app_lib = AppLib()
app_lib.update_path()
# import modules after path has been updated
# third-party
from app import App # pylint: disable=import-outside-toplevel
from tcex import TcEx # pylint: disable=import-outside-toplevel
tcex = TcEx()
try:
# load App class
app = App(tcex)
# perform prep/setup operations
app.setup(**{})
# run the App logic
app.run(**{})
# perform cleanup/teardown operations
app.teardown(**{})
# explicitly call the exit method
tcex.exit(msg=app.exit_message)
except Exception as e:
main_err = f'Generic Error. See logs for more details ({e}).'
tcex.log.error(traceback.format_exc())
tcex.playbook.exit(1, main_err)
def app_init(self, args):
"""Return an instance of App."""
from app import App # pylint: disable=import-error
# return App(self.get_tcex(args))
args = args or {}
# update App paths
args['tc_in_path'] = os.path.join(self.default_args.get('tc_in_path'),
self.test_case_feature)
args['tc_log_path'] = os.path.join(
self.default_args.get('tc_log_path'), self.test_case_feature,
self.test_case_name)
args['tc_out_path'] = os.path.join(
self.default_args.get('tc_out_path'), self.test_case_feature,
self.test_case_name)
args['tc_temp_path'] = os.path.join(
self.default_args.get('tc_temp_path'), self.test_case_feature,
self.test_case_name)
# update default args with app args
app_args = dict(self.default_args)
app_args.update(args)
# app_args['tc_log_file'] = '{}.log'.format(self.test_case_name)
app_args['tc_logger_name'] = self.context
if self.install_json.get('runtimeLevel').lower() in [
'triggerservice',
'webhooktriggerservice',
]:
# service Apps will get their args/params from encrypted file in the "in" directory
data = json.dumps(app_args, sort_keys=True).encode('utf-8')
key = ''.join(
random.choice(string.ascii_lowercase) for i in range(16))
fp = FileParams()
fp.EVP_EncryptInit(fp.EVP_aes_128_cbc(), key.encode('utf-8'),
b'\0' * 16)
encrypted_data = fp.EVP_EncryptUpdate(data) + fp.EVP_EncryptFinal()
app_params_json = os.path.join(self.test_case_feature_dir,
'.app_params.json')
with open(app_params_json, 'wb') as fh:
fh.write(encrypted_data)
# create environment variable for tcex inputs method to pick up to read encrypted file
os.environ['TC_APP_PARAM_KEY'] = key
os.environ['TC_APP_PARAM_FILE'] = app_params_json
# tcex will read args/params from encrypted file
tcex = TcEx()
else:
tcex = TcEx(config=app_args)
return App(tcex)
def app_init(self, args):
"""Return an instance of App."""
from app import App # pylint: disable=import-error
# return App(self.get_tcex(args))
args = args or {}
# update App paths
args['tc_in_path'] = os.path.join(self.default_args.get('tc_in_path'),
self.test_case_feature)
args['tc_log_path'] = os.path.join(
self.default_args.get('tc_log_path'), self.test_case_feature,
self.test_case_name)
args['tc_out_path'] = os.path.join(
self.default_args.get('tc_out_path'), self.test_case_feature,
self.test_case_name)
args['tc_temp_path'] = os.path.join(
self.default_args.get('tc_temp_path'), self.test_case_feature,
self.test_case_name)
# update default args with app args
app_args = dict(self.default_args)
app_args.update(args)
# app_args['tc_log_file'] = '{}.log'.format(self.test_case_name)
app_args['tc_logger_name'] = self.context
tcex = TcEx(config=app_args)
return App(tcex)
def test_file_params():
"""Test encrypted file params input."""
os.environ['TC_APP_PARAM_FILE'] = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'app_params.aes'
)
os.environ['TC_APP_PARAM_KEY'] = 'NdwGwTuWjEXyuLLE'
# clear sys.argv to avoid invalid arguments
sys_argv_orig = sys.argv
sys.argv = sys.argv[:1]
# initialize tcex and add required argument
fp_tcex = TcEx()
# assert args
assert fp_tcex.args.tc_api_path == 'https://localhost:8443/api'
assert fp_tcex.args.tc_svc_server_topic == 'svc-server-c1fb3cd06ff44e814fa87e4b928fe13d'
assert fp_tcex.args.service_id == 'c1fb3cd06ff44e814fa87e4b928fe13d'
# reset sys.argv
sys.argv = sys_argv_orig
try:
del os.environ['TC_APP_PARAM_FILE']
del os.environ['TC_APP_PARAM_KEY']
except Exception:
pass
def setup_method(self):
"""Run before each test method runs."""
self._timer_method_start = time.time()
self._current_test = os.getenv('PYTEST_CURRENT_TEST').split(' ')[0]
self.log.title(self._current_test, '=')
self.log.data('setup method', 'started', datetime.now().isoformat())
# create and log current context
self.context = os.getenv('TC_PLAYBOOK_DB_CONTEXT', str(uuid.uuid4()))
self.log.data('setup method', 'context', self.context)
# setup per method instance of tcex
args = self.default_args.copy()
args['tc_log_file'] = os.path.join(self.test_case_log_test_dir,
'setup.log')
args[
'tc_logger_name'] = f'tcex-{self.test_case_feature}-{self.test_case_name}'
self.tcex = TcEx(config=args)
# initialize new stager instance
self._stager = self.stager_init()
# initialize new validator instance
self._validator = self.validator_init()
# Adding this for batch to created the -batch and errors files
os.makedirs(os.path.join(self.test_case_log_test_dir, 'DEBUG'),
exist_ok=True)
def test_add_inputs(config_data, tc_log_file):
"""Test args.
Test argument_parser add_argument method with a required field and config_data.
"""
# update config data
config_data['name'] = 'pytest'
# test apps that use logging over tc_log_level
config_data['logging'] = config_data.pop('tc_log_level')
config_data['tc_log_file'] = tc_log_file
config_data['tc_log_to_api'] = True
# clear sys.argv to avoid invalid arguments
sys_argv_orig = sys.argv
sys.argv = sys.argv[:1]
# initialize tcex and add required argument
tcex = TcEx(config=config_data)
tcex.parser.add_argument('--name', required=True)
# parse args
args = tcex.args
assert args.name == config_data.get('name')
# reset sys.argv
sys.argv = sys_argv_orig
def tcex():
"""Return an instance of tcex."""
# create log structure for feature/test (e.g., args/test_args.log)
config_data_ = dict(_config_data)
config_data_['tc_log_file'] = _tc_log_file()
# clear sys.argv to avoid invalid arguments
sys.argv = sys.argv[:1]
return TcEx(config=config_data_)
def run(**kwargs):
"""Update path and run the App."""
# update the path to ensure the App has access to required modules
app_lib = AppLib()
app_lib.update_path()
# import modules after path has been updated
from tcex import TcEx # pylint: disable=import-outside-toplevel
from app import App # pylint: disable=import-outside-toplevel
tcex = TcEx()
try:
# load App class
app = App(tcex)
# set app property in testing framework
if callable(kwargs.get('set_app')):
kwargs.get('set_app')(app)
# configure custom trigger message handler
tcex.service.create_config_callback = app.create_config_callback
tcex.service.delete_config_callback = app.delete_config_callback
tcex.service.shutdown_callback = app.shutdown_callback
tcex.service.webhook_event_callback = app.webhook_event_callback
# perform prep/setup operations
app.setup()
# listen on channel/topic
tcex.service.listen()
# start heartbeat threads
tcex.service.heartbeat()
# inform TC that micro-service is Ready
tcex.service.ready = True
# loop until exit
if hasattr(app, 'loop_forever'):
app.loop_forever() # pylint: disable=no-member
else:
tcex.log.info('Looping until shutdown')
while tcex.service.loop_forever(sleep=1):
pass
# perform cleanup/teardown operations
app.teardown()
# explicitly call the exit method
tcex.playbook.exit(msg=app.exit_message)
except Exception as e:
main_err = f'Generic Error. See logs for more details ({e}).'
tcex.log.error(traceback.format_exc())
tcex.playbook.exit(1, main_err)
def test_api_handler(self, config_data): # pylint: disable=no-self-use
"""Test API logging handler"""
config_data['tc_log_to_api'] = True
tcex = TcEx(config=config_data)
for _ in range(0, 20):
tcex.log.trace('TRACE LOGGING')
tcex.log.debug('DEBUG LOGGING')
tcex.log.info('INFO LOGGING')
tcex.log.warning('WARNING LOGGING')
tcex.log.error('ERROR LOGGING')
def test_no_valid_credentials(self, config_data):
"""Testing initializing tcex with no credentials."""
hmac_config_data = dict(config_data)
del hmac_config_data['tc_token']
del hmac_config_data['tc_token_expires']
hmac_tcex = TcEx(config=hmac_config_data)
hmac_tcex.args # pylint: disable=pointless-statement
r = hmac_tcex.session.get('/v2/owners')
assert r.status_code == 200
def app_init(self, args):
"""Return an instance of App."""
from app import App # pylint: disable=import-error
# return App(self.get_tcex(args))
args = args or {}
# update path args
self._update_path_args(args)
# update default args with app args
app_args = dict(self.default_args)
app_args.update(args)
# app_args['tc_log_file'] = f'{self.test_case_name}.log'
app_args['tc_logger_name'] = self.context
if self.install_json.get('runtimeLevel').lower() in [
'triggerservice',
'webhooktriggerservice',
]:
# service Apps will get their args/params from encrypted file in the "in" directory
data = json.dumps(app_args, sort_keys=True).encode('utf-8')
key = ''.join(
random.choice(string.ascii_lowercase) for i in range(16))
encrypted_data = self._encrypt_file_contents(key, data)
app_params_json = os.path.join(self.test_case_feature_dir,
'.app_params.json')
with open(app_params_json, 'wb') as fh:
fh.write(encrypted_data)
# create environment variable for tcex inputs method to pick up to read encrypted file
os.environ['TC_APP_PARAM_KEY'] = key
os.environ['TC_APP_PARAM_FILE'] = app_params_json
# tcex will read args/params from encrypted file
tcex = TcEx()
else:
tcex = TcEx(config=app_args)
return App(tcex)
def tcex_proxy_external():
"""Return an instance of tcex.
mitmproxy -p 4242 --ssl-insecure
"""
# create log structure for feature/test (e.g., args/test_args.log)
config_data_ = dict(_config_data)
config_data_['tc_proxy_external'] = True
config_data_['tc_proxy_host'] = 'localhost'
config_data_['tc_proxy_port'] = '4242'
# clear sys.argv to avoid invalid arguments
sys.argv = sys.argv[:1]
return TcEx(config=config_data_)
def setup_method(self):
"""Setup before each test case is called."""
_config_data['arg1'] = 'arg1'
_config_data['fail_on_enabled'] = True
_config_data['fail_on_disabled'] = False
_config_data['fail_on_string'] = 'false'
# _config_data['iterate_on'] = ['one', 'two', 'three']
_config_data['iterate_on'] = 'iterate_on'
_config_data['none'] = None
_config_data['null'] = 'null'
_config_data['tc_log_file'] = self.tc_log_file
sys.argv = sys.argv[:1]
self.tcex = TcEx(config=_config_data)
self.args = self.tcex.args
def run():
"""Update path and run the App."""
# update the path to ensure the App has access to required modules
app_lib = AppLib()
app_lib.update_path()
# import modules after path has been updated
from tcex import TcEx
from app import App
config_file = 'app_config.json'
if not os.path.isfile(config_file):
print(f'Missing {config_file} config file.')
tcex = TcEx(config_file=config_file)
try:
# load App class
app = App(tcex)
# perform prep/setup operations
app.setup()
# run the App logic
app.run()
# perform cleanup/teardown operations
app.teardown()
# explicitly call the exit method
tcex.exit(msg=app.exit_message)
except Exception as e:
main_err = f'Generic Error. See logs for more details ({e}).'
tcex.log.error(traceback.format_exc())
tcex.playbook.exit(1, main_err)
def tcex(self):
"""Return an instance of tcex."""
# clear sys.argv to avoid invalid arguments
if self.clear_argv:
sys.argv = sys.argv[:1]
tcex = TcEx(config=self.config_data)
# cleanup environment variables
if os.getenv('TC_APP_PARAM_FILE', None):
del os.environ['TC_APP_PARAM_FILE']
if os.getenv('TC_APP_PARAM_KEY', None):
del os.environ['TC_APP_PARAM_KEY']
return tcex
def validator_init(self):
"""Return instance of Stager class."""
tc_log_file = os.path.join(self.test_case_log_test_dir, 'validate.log')
# args data
args = self.default_args.copy()
# override default log level if profiled
args['tc_log_level'] = 'warning'
# set log path to be the feature and test case name
args['tc_log_file'] = tc_log_file
# set a logger name to have a logger specific for stager
args['tc_logger_name'] = 'tcex-validator'
tcex = TcEx(config=args)
return Validator(tcex, logger)
def stager_init(self):
"""Return instance of Stager class."""
tc_log_file = os.path.join(self.test_case_feature, self.test_case_name,
'stage.log')
# args data
args = dict(self.default_args)
# override default log level if profiled
args['tc_log_level'] = 'warning'
# set log path to be the feature and test case name
args['tc_log_file'] = tc_log_file
# set a logger name to have a logger specific for stager
args['tc_logger_name'] = 'tcex-stager'
tcex = TcEx(config=args)
return Stager(tcex, logger, self.log_data)
def test_aot_inputs(tcex, config_data, tc_log_file):
"""Test inputs.config_file() method."""
tc_action_channel = 'pytest-action-channel'
# add custom config data
config_data['my_bool'] = 'true'
config_data['my_multi'] = 'one|two'
# send AOT message
aot_msg = {'type': 'execute', 'params': config_data}
tcex.playbook.db.rpush(tc_action_channel, json.dumps(aot_msg))
# update sys.argv to enable aot
sys_argv_orig = sys.argv
sys.argv = sys.argv[:1] + [
'--tc_aot_enabled',
'--tc_action_channel',
tc_action_channel,
'--tc_log_file',
tc_log_file,
]
my_tcex = TcEx()
# add custom args (install.json defined in conftest.py)
my_tcex.parser.add_argument('--my_bool', action='store_true')
my_tcex.parser.add_argument('--my_multi', action='append')
# parser args
args = my_tcex.args
rargs = my_tcex.rargs
assert my_tcex.inputs.params.my_bool is True
assert my_tcex.inputs.resolved_params.my_bool is True
assert args.my_bool is True
assert rargs.my_bool is True # pylint: disable=no-member
assert args.my_multi == ['one', 'two']
assert rargs.my_multi == ['one', 'two'] # pylint: disable=no-member
# reset sys.argv
sys.argv = sys_argv_orig
def test_secure_params(config_data, tc_log_file, monkeypatch):
"""Test get_secure_params method."""
# add custom config data
config_data['my_bool'] = 'true'
config_data['my_multi'] = 'one|two'
# monkeypatch session get method
def get_secure_params(self): # pylint: disable=unused-argument
return config_data
monkeypatch.setattr(Inputs, '_get_secure_params', get_secure_params)
# update sys.argv to enable secure_params
sys_argv_orig = sys.argv
sys.argv = sys.argv[:1] + [
'--tc_secure_params',
'--tc_token',
config_data.get('tc_token'),
'--tc_token_expires',
config_data.get('tc_token_expires'),
'--tc_log_file',
tc_log_file,
]
tcex = TcEx()
# add custom args (install.json defined in conftest.py)
tcex.parser.add_argument('--my_bool', action='store_true')
tcex.parser.add_argument('--my_multi', action='append')
# parser args
args = tcex.args
rargs = tcex.rargs
assert args.my_bool is True
assert rargs.my_bool is True # pylint: disable=no-member
assert args.my_multi == ['one', 'two']
assert rargs.my_multi == ['one', 'two'] # pylint: disable=no-member
# reset sys.argv
sys.argv = sys_argv_orig
def get_tcex(self, args=None):
"""Return an instance of App."""
args = args or {}
app_args = self.default_args
app_args.update(args)
if self.tcex is not None and self.context == self.tcex.default_args.tc_playbook_db_context:
self.tcex.tcex_args.inject_params(
app_args) # during run this is required
return self.tcex
sys.argv = [
sys.argv[0],
'--tc_log_path',
'log',
'--tc_log_file',
'{}/app.log'.format(self.context),
]
self.tcex = TcEx()
# TODO: validate this
self.tcex.logger.update_handler_level('error')
self.tcex.tcex_args.inject_params(app_args) # required for stager
return self.tcex
def __init__(self, access_key: str, secret_key: str, api_path: str):
""" Initialize client configuration:
Args:
access_key: Generated access key.
secret_key: Generated secret key.
api_path: https://api.threatconnect.com
References:
1. HMAC: https://docs.threatconnect.com/en/latest/tcex/authorization.html
2. Creating user: https://training.threatconnect.com/learn/article/creating-user-accounts-kb-article#2
Notes:
1. When importing TcEx, Print occurred therefor an error raised in the server due to not valid stdout.
"""
with suppress_stdout():
from tcex import TcEx
self._client = TcEx(
config={
"api_access_id": access_key,
"api_secret_key": secret_key,
"tc_api_path": api_path,
})
def setup_method(self):
"""Run before each test method runs."""
self._timer_method_start = time.time()
self._current_test = os.getenv('PYTEST_CURRENT_TEST').split(' ')[0]
self.log.info('{0} {1} {0}'.format('=' * 10, self._current_test))
self.log_data('setup method', 'started', datetime.now().isoformat())
# create and log current context
self.context = os.getenv('TC_PLAYBOOK_DB_CONTEXT', str(uuid.uuid4()))
self.log_data('setup method', 'context', self.context)
# setup per method instance of tcex
args = dict(self.default_args)
args['tc_log_file'] = os.path.join(self.test_case_feature,
self.test_case_name, 'setup.log')
args['tc_logger_name'] = 'tcex-{}-{}'.format(self.test_case_feature,
self.test_case_name)
self.tcex = TcEx(config=args)
# initialize new stager instance
self._stager = self.stager_init()
# initialize new validator instance
self._validator = self.validator_init()
sys.path.remove(cwd)
except ValueError:
pass
sys.path.insert(0, cwd)
if __name__ == '__main__':
# update the path to ensure the App has access to required modules
_update_app_path()
# import modules after path has been updated
from tcex import TcEx
from app import App
tcex = TcEx()
try:
# load App class
app = App(tcex)
# perform prep/setup operations
app.setup()
# run the App logic
app.run()
# perform cleanup/teardown operations
app.teardown()
# explicitly call the exit method
def __init__(self, owner=None):
self.owner = owner
self.tcex = TcEx()
self._authenticate()
self.default_metadata = {}
self.inflect_engine = inflect.engine()
sys.path.remove(cwd)
except ValueError:
pass
sys.path.insert(0, cwd)
if __name__ == '__main__':
# update the path to ensure the App has access to required modules
_update_app_path()
# import modules after path has been updated
from tcex import TcEx
from app import App
tcex = TcEx()
try:
# load App class
app = App(tcex)
# parse args
app.parse_args()
# perform prep/startup operations
app.start()
# run the App logic
if hasattr(app.args, 'tc_action') and app.args.tc_action is not None:
# if the args NameSpace has the reserved arg of "tc_action", this arg value is used to
# triggers a call to the app.<tc_action>() method. an exact match to the method is
"""Parse the incoming PDF contents as a PDF."""
# handle the incoming arguments
args = parse_arguments()
# read the pdf_content from the tcex playbook to get the actual value of the pdf content
pdf_content = tcex.playbook.read(args.pdf_content)
# get the text from the PDF contents
text = convert_pdf_to_txt(pdf_content)
# output the text as a variable for downstream apps
tcex.playbook.create_output('pdf_reader.text', text)
# hasta luego
tcex.exit()
if __name__ == "__main__":
# initialize a TcEx instance
tcex = TcEx()
try:
# start the app
main()
except SystemExit:
pass
except: # if there are any strange errors, log it to the logging in the UI
err = 'Generic Error. See logs for more details.'
tcex.log.error(traceback.format_exc())
tcex.message_tc(err)
tcex.playbook.exit(1)
# -*- coding: utf-8 -*-
""" JSON Pretty Playbook App """
import codecs
import json
import traceback
import sys
from tcex import TcEx
# Python 2 unicode
if sys.version_info[0] == 2:
reload(sys)
sys.setdefaultencoding('utf-8')
tcex = TcEx()
# App args
tcex.parser.add_argument('--json_data', required=True)
args = tcex.args
def main():
"""Main App Logic"""
json_data = tcex.playbook.read(args.json_data)
try:
json.loads(json_data)
except Exception as e:
err = 'JSON data was not properly formatted ({}).'.format(e)
tcex.log.error(err)
tcex.message_tc(err)
from app_lib import AppLib
# pylint: disable=no-member
if __name__ == '__main__':
# update the path to ensure the App has access to required modules
app_lib = AppLib()
app_lib.update_path()
# import modules after path has been updated
from tcex import TcEx
from app import App
tcex = TcEx()
try:
# load App class
app = App(tcex)
# configure custom trigger message handler
tcex.service.create_config_callback = app.create_config_callback
tcex.service.delete_config_callback = app.delete_config_callback
tcex.service.shutdown_callback = app.shutdown_callback
# perform prep/setup operations
app.setup()
# listen on channel/topic
tcex.service.listen()
def test_stream_handler(config_data): # pylint: disable=no-self-use
"""Test API logging handler"""
tcex = TcEx(config=config_data)
handler_name = 'pytest-sh'
tcex.logger.add_stream_handler(name=handler_name, level='trace')
tcex.logger.remove_handler_by_name(handler_name)
