def __create_incident_with_playbook(client, name, playbook_id):
# create incident
kwargs = {'createInvestigation': True, 'playbookId': playbook_id}
try:
r = client.CreateIncident(name, None, None, None, None, None, None,
**kwargs)
except RuntimeError as err:
print_error(str(err))
response_json = r.json()
inc_id = response_json['id']
# get incident
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
# poll up to 1 second
timeout = time.time() + 10
while incidents['total'] != 1:
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
if time.time() > timeout:
print_error('failed to get incident with id:' + inc_id)
return False, -1
time.sleep(1)
return incidents['data'][0], inc_id
def print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests,
skipped_integration):
succeed_count = len(succeed_playbooks)
failed_count = len(failed_playbooks)
skipped_count = len(skipped_tests)
print('\nTEST RESULTS:')
print('\t Number of playbooks tested - ' +
str(succeed_count + failed_count))
print_color('\t Number of succeeded tests - ' + str(succeed_count),
LOG_COLORS.GREEN)
if len(skipped_integration) > 0:
print_warning('\t Number of skipped integration - ' +
str(len(skipped_integration)) + ':')
for playbook_id in skipped_integration:
print_warning('\t - ' + playbook_id)
if skipped_count > 0:
print_warning('\t Number of skipped tests - ' + str(skipped_count) +
':')
for playbook_id in skipped_tests:
print_warning('\t - ' + playbook_id)
if failed_count > 0:
print_error('\t Number of failed tests - ' + str(failed_count) + ':')
for playbook_id in failed_playbooks:
print_error('\t - ' + playbook_id)
def __delete_integration_instance(client, instance_id):
res = client.req('DELETE', '/settings/integration/' + urllib.quote(instance_id), {})
if res.status_code is not 200:
print_error('delete integration instance failed\nStatus code' + str(res.status_code))
print_error(pformat(res.json()))
return False
return True
def set_integration_params(demisto_api_key, integrations, secret_params, instance_names, playbook_id):
for integration in integrations:
integration_params = [item for item in secret_params if item['name'] == integration['name']]
if integration_params:
matched_integration_params = integration_params[0]
if len(integration_params) != 1:
found_matching_instance = False
for item in integration_params:
if item.get('instance_name', 'Not Found') in instance_names:
matched_integration_params = item
found_matching_instance = True
if not found_matching_instance:
optional_instance_names = [optional_integration.get('instance_name') for optional_integration in
integration_params]
print_error(FAILED_MATCH_INSTANCE_MSG.format(playbook_id, len(integration_params),
integration['name'],
'\n'.join(optional_instance_names)))
return False
integration['params'] = matched_integration_params.get('params', {})
integration['byoi'] = matched_integration_params.get('byoi', True)
elif 'Demisto REST API' == integration['name']:
integration['params'] = {
'url': 'https://localhost',
'apikey': demisto_api_key,
'insecure': True,
}
return True
def set_integration_params(demisto_api_key, integrations, secret_params, instance_names, playbook_id):
for integration in integrations:
integration_params = [item for item in secret_params if item['name'] == integration['name']]
if integration_params:
matched_integration_params = integration_params[0]
if len(integration_params) != 1:
found_matching_instance = False
for item in integration_params:
if item.get('instance_name', 'Not Found') in instance_names:
matched_integration_params = item
found_matching_instance = True
if not found_matching_instance:
optional_instance_names = [optional_integration.get('instance_name') for optional_integration in integration_params]
print_error("{} Failed to run\n, There are {} instances of {},"
"please select of them by using the instance_name "
"argument in conf.json the options are:\n{}".format(playbook_id, len(integration_params),
integration['name'], '\n'.join(optional_instance_names)))
return False
integration['params'] = matched_integration_params.get('params', {})
integration['byoi'] = matched_integration_params.get('byoi', True)
elif 'Demisto REST API' == integration['name']:
integration['params'] = {
'url': 'https://localhost',
'apikey': demisto_api_key,
'insecure': True,
}
return True
def print_test_summary(succeed_playbooks, failed_playbooks):
succeed_count = len(succeed_playbooks)
failed_count = len(failed_playbooks)
print('\nTEST RESULTS:')
print('\t Number of playbooks tested - ' +
str(succeed_count + failed_count))
print_color('\t Number of succeeded tests - ' + str(succeed_count),
LOG_COLORS.GREEN)
if len(failed_playbooks) > 0:
print_error('\t Number of failed tests - ' + str(failed_count) + ':')
for playbook_id in failed_playbooks:
print_error('\t - ' + playbook_id)
def __delete_incident(client, incident):
res = client.req('POST', '/incident/batchDelete', {
'ids': [incident['id']],
'filter': {},
'all': False
})
if res.status_code is not 200:
print_error('delete incident failed\nStatus code' + str(res.status_code))
print_error(pformat(res.json()))
return False
return True
def __test_integration_instance(client, module_instance):
res = client.req('POST', '/settings/integration/test', module_instance)
if res.status_code != 200:
print_error('Integration-instance test ("Test" button) failed.\nBad status code: ' + str(res.status_code))
return False
result_object = res.json()
success = result_object['success']
if not success:
print_error('Test integration failed.\n Failure message: ' + result_object['message'])
return False
return True
def __get_integration_config(client, integration_name):
res = client.req('POST', '/settings/integration/search', {
'page': 0, 'size': 100, 'query': 'name:' + integration_name
})
res = res.json()
all_configurations = res['configurations']
match_configurations = [x for x in all_configurations if x['name'] == integration_name]
if not match_configurations or len(match_configurations) == 0:
print_error('integration was not found')
return None
return match_configurations[0]
def __create_incident_with_playbook(client, name, playbook_id):
# create incident
kwargs = {'createInvestigation': True, 'playbookId': playbook_id}
try:
r = client.CreateIncident(name, None, None, None, None,
None, None, **kwargs)
except RuntimeError as err:
print_error(str(err))
response_json = r.json()
inc_id = response_json['id']
# get incident
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
# poll up to 1 second
timeout = time.time() + 10
while incidents['total'] != 1:
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
if time.time() > timeout:
if inc_id == 'incCreateErr':
print_error('Failed to create incident. Possible reasons are:\nMismatch between playbookID in conf.json and the id of the real playbook you were trying to use, or schema problems in the TestPlaybook.')
return False, -1
print_error('failed to get incident with id:' + inc_id)
return False, -1
time.sleep(1)
return incidents['data'][0], inc_id
def __create_incident_with_playbook(client, name, playbook_id):
# create incident
kwargs = {'createInvestigation': True, 'playbookId': playbook_id}
response_json = {}
try:
r = client.CreateIncident(name, None, None, None, None, None, None,
**kwargs)
response_json = r.json()
except RuntimeError as err:
print_error(str(err))
inc_id = response_json.get('id', 'incCreateErr')
if inc_id == 'incCreateErr':
print_error(INC_CREATION_ERR)
return False, -1
# get incident
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
# poll the incidents queue for a max time of 25 seconds
timeout = time.time() + 25
while incidents['total'] != 1:
incidents = client.SearchIncidents(0, 50, 'id:' + inc_id)
if time.time() > timeout:
print_error('Got timeout for searching incident with id {}, '
'got {} incidents in the search'.format(
inc_id, incidents['total']))
return False, -1
time.sleep(1)
return incidents['data'][0], inc_id
def __disable_integrations_instances(client, module_instances):
for configured_instance in module_instances:
# tested with POSTMAN, this is the minimum required fields for the request.
module_instance = {
key: configured_instance[key]
for key in [
'id',
'brand',
'name',
'data',
'isIntegrationScript',
]
}
module_instance['enable'] = "false"
module_instance['version'] = -1
res = client.req('PUT', '/settings/integration', module_instance)
if res.status_code != 200:
print_error('disable instance failed with status code ' +
str(res.status_code))
print_error(pformat(res.json()))
def __print_investigation_error(client, playbook_id, investigation_id):
res = client.req('POST', '/investigation/' + urllib.quote(investigation_id), {})
if res.status_code == 200:
entries = res.json()['entries']
print_error('Playbook ' + playbook_id + ' has failed:')
for entry in entries:
if entry['type'] == ENTRY_TYPE_ERROR:
if entry['parentContent']:
print_error('\t- Command: ' + str(entry['parentContent']))
print_error('\t- Body: ' + str(entry['contents']))
def test_integration(client, integrations, playbook_id, options={}):
# create integrations instances
instance_ids = []
for integration in integrations:
integration_name = integration.get('name', None)
integration_params = integration.get('params', None)
is_byoi = integration.get('byoi', True)
instance_id = __create_integration_instance(client, integration_name, integration_params, is_byoi)
if not instance_id:
print_error('Failed to create instance')
__delete_integrations_instances(client, instance_ids)
return False
instance_ids.append(instance_id)
print('Create integration %s succeed' % (integration_name, ))
# create incident with playbook
incident = __create_incident_with_playbook(client, 'inc_%s' % (playbook_id, ), playbook_id)
if not incident:
return False
investigation_id = incident['investigationId']
if investigation_id is None or len(investigation_id) == 0:
print_error('Failed to get investigation id of incident:' + incident)
return False
timeout_amount = options['timeout'] if 'timeout' in options else DEFAULT_TIMEOUT
timeout = time.time() + timeout_amount
interval = options['interval'] if 'interval' in options else DEFAULT_INTERVAL
i = 1
# wait for playbook to finish run
while True:
# give playbook time to run
time.sleep(interval)
# fetch status
playbook_state = __get_investigation_playbook_state(client, investigation_id)
if playbook_state == PB_Status.COMPLETED:
break
if playbook_state == PB_Status.FAILED:
print_error(playbook_id + ' failed with error/s')
__print_investigation_error(client, playbook_id, investigation_id)
break
if time.time() > timeout:
print_error(playbook_id + ' failed on timeout')
break
print 'loop no.' + str(i) + ', playbook state is ' + playbook_state
i = i + 1
test_pass = playbook_state == PB_Status.COMPLETED
if test_pass:
# delete incident
__delete_incident(client, incident)
# delete integration instance
__delete_integrations_instances(client, instance_ids)
return test_pass
def main():
options = options_handler()
username = options.user
password = options.password
server = options.server
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
slack = options.slack
CircleCI = options.circleci
buildNumber = options.buildNumber
build_name = options.buildName
if not (username and password and server):
print_error('You must provide server user & password arguments')
sys.exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code != 200:
print_error("Login has failed with status code " + str(res.status_code))
sys.exit(1)
demisto_api_key = generate_demisto_api_key(c)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nigthly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filterd_tests, is_filter_configured, run_all_tests = extract_filtered_tests()
if is_filter_configured and not run_all_tests:
is_nightly = True
if not tests or len(tests) == 0:
print('no integrations are configured for test')
return
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
for t in tests:
playbook_id = t['playbookID']
nightly_test = t.get('nightly', False)
integrations_conf = t.get('integrations', [])
instance_names_conf = t.get('instance_names', [])
test_message = 'playbook: ' + playbook_id
test_options = {
'timeout': t.get('timeout', default_test_timeout)
}
if not isinstance(integrations_conf, list):
integrations_conf = [integrations_conf, ]
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf, ]
has_skipped_integration, integrations, is_nightly_integration = collect_integrations(
integrations_conf, skipped_integration, skipped_integrations_conf, nightly_integrations)
skip_nightly_test = True if (nightly_test or is_nightly_integration) and not is_nightly else False
# Skip nightly test
if skip_nightly_test:
print '------ Test %s start ------' % (test_message,)
print 'Skip test'
print '------ Test %s end ------' % (test_message,)
continue
if not run_all_tests:
# Skip filtered test
if is_filter_configured and playbook_id not in filterd_tests:
continue
# Skip bad test
if playbook_id in skipped_tests_conf.keys():
skipped_tests.add("{0} - reason: {1}".format(playbook_id, skipped_tests_conf[playbook_id]))
continue
# Skip integration
if has_skipped_integration:
continue
are_params_set = set_integration_params(demisto_api_key, integrations,
secret_params, instance_names_conf, playbook_id)
if not are_params_set:
failed_playbooks.append(playbook_id)
continue
test_message = update_test_msg(integrations, test_message)
run_test(c, failed_playbooks, integrations, playbook_id,
succeed_playbooks, test_message, test_options, slack, CircleCI,
buildNumber, server, build_name)
print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration)
create_result_files(failed_playbooks, skipped_integration, skipped_tests)
os.remove(FILTER_CONF)
if len(failed_playbooks):
with open("./Tests/is_build_failed.txt", "w") as is_build_failed_file:
is_build_failed_file.write('Build failed')
sys.exit(1)
def main():
options = options_handler()
username = options.user
password = options.password
server = options.server
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
slack = options.slack
CircleCI = options.circleci
buildNumber = options.buildNumber
if not (username and password and server):
print_error('You must provide server user & password arguments')
sys.exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code is not 200:
print_error("Login has failed with status code " +
str(res.status_code))
sys.exit(1)
demisto_api_key = generate_demisto_api_key(c)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
skipped_integrations_conf = conf['skipped_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filterd_tests, is_filter_configured = extract_filtered_tests()
if is_filter_configured:
is_nightly = True
if not tests or len(tests) is 0:
print('no integrations are configured for test')
return
skipped_tests = []
failed_playbooks = []
succeed_playbooks = []
skipped_integration = []
for t in tests:
playbook_id = t['playbookID']
nightly_test = t.get('nightly', False)
integrations_conf = t.get('integrations', [])
skip_test = True if nightly_test and not is_nightly else False
test_message = 'playbook: ' + playbook_id
test_options = {
'timeout':
t['timeout'] if 'timeout' in t else conf.get('testTimeout', 30),
'interval':
conf.get('testInterval', 10)
}
if not isinstance(integrations_conf, list):
integrations_conf = [integrations_conf]
has_skipped_integration, integrations = collect_integrations(
integrations_conf, is_filter_configured, skipped_integration,
skipped_integrations_conf)
# Skip nightly test
if skip_test:
print '------ Test %s start ------' % (test_message, )
print 'Skip test'
print '------ Test %s end ------' % (test_message, )
continue
# Skip filtered test
if is_filter_configured and playbook_id not in filterd_tests:
continue
# Skip bad test
if playbook_id in skipped_tests_conf:
skipped_tests.append(playbook_id)
continue
# Skip integration
if has_skipped_integration:
continue
set_integration_params(demisto_api_key, integrations, secret_params)
test_message = update_test_msg(integrations, test_message)
run_test(c, failed_playbooks, integrations, playbook_id,
succeed_playbooks, test_message, test_options, slack,
CircleCI, buildNumber, server)
print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests,
skipped_integration)
create_result_files(failed_playbooks, skipped_integration, skipped_tests)
os.remove(FILTER_CONF)
if len(failed_playbooks):
sys.exit(1)
def main():
options = options_handler()
username = options.user
password = options.password
server = options.server
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
if not (username and password and server):
print_error('You must provide server user & password arguments')
exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code is not 200:
print_error("Login has failed with status code " +
str(res.status_code))
exit(1)
with open(conf_path) as data_file:
conf = json.load(data_file)
secret_conf = None
if secret_conf_path:
with open(secret_conf_path) as data_file:
secret_conf = json.load(data_file)
integrations = conf['integrations']
secret_integrations = secret_conf['integrations'] if secret_conf else []
if not integrations or len(integrations) is 0:
print('no integrations are configured for test')
return
succeed_playbooks = []
failed_playbooks = []
for integration in integrations:
test_options = {
'timeout':
integration['timeout'] if 'timeout' in integration else conf.get(
'testTimeout', 30),
'interval':
conf.get('testInterval', 10)
}
integration_name = integration.get('name', None)
playbook_id = integration['playbookID']
if 'params' in integration:
integration_params = integration['params']
else:
# get from secret conf
secret_integration_match = [
item for item in secret_integrations
if item["name"] == integration_name
]
if len(secret_integration_match) > 0:
integration_params = secret_integration_match[0].get('params')
else:
integration_params = {}
if integration_name:
test_message = 'integration: ' + integration_name + ' with playbook: ' + playbook_id
else:
test_message = 'playbook: ' + playbook_id
print '------ Test %s start ------' % (test_message, )
nightly_test = integration.get('nightly', False)
is_byoi = integration.get('byoi', True)
skip_test = True if nightly_test and not is_nightly else False
if skip_test:
print 'Skip test'
else:
# run test
succeed = test_integration(c, integration_name, integration_params,
playbook_id, is_byoi, test_options)
# use results
if succeed:
print 'PASS: %s succeed' % (test_message, )
succeed_playbooks.append(playbook_id)
else:
print 'Failed: %s failed' % (test_message, )
failed_playbooks.append(playbook_id)
print '------ Test %s end ------' % (test_message, )
print_test_summary(succeed_playbooks, failed_playbooks)
if len(failed_playbooks):
sys.exit(1)
def run_end_to_end_test(network: str, plz_host: str, plz_port: int,
test_name: str, bless: bool) -> bool:
"""
:param network: docker network where the controller is running (or `host`)
:param plz_host: host where the controller is running
:param plz_port: port where controller is listening
:param test_name: name of the test to run (path from test/)
:param bless: whether to override the expected output and logs with the
results of the run
:return: whether the test passed
"""
if not os.path.isdir(DATA_DIRECTORY):
os.mkdir(DATA_DIRECTORY)
# Make sure the directory has a single slash at the end
test_directory = os.path.join(
os.path.normpath(os.path.join(TEST_DIRECTORY, test_name)), '')
test_utils.print_info(f'Running {test_name}...')
if os.path.isfile(f'{test_directory}/expected-status'):
with open(f'{test_directory}/expected-status', 'r') as f:
expected_exit_status = int(f.read())
else:
expected_exit_status = 0
expected_logs_file_name = f'{test_directory}/expected-logs'
expected_output_directory = f'{test_directory}/expected-output'
with \
NamedTemporaryFile(
prefix='plz-test-logs_',
dir=DATA_DIRECTORY,
mode='wb') as logs_file, \
TemporaryDirectory(
prefix='plz-test-output_',
dir=DATA_DIRECTORY) as output_directory:
output_directory_name = os.path.abspath(output_directory)
start = datetime.datetime.now()
actual_exit_status = run_cli(
network=network,
plz_host=plz_host,
plz_port=plz_port,
test_name=test_name,
app_directory=test_directory,
actual_logs_file=logs_file,
output_directory_name=output_directory_name)
end = datetime.datetime.now()
test_utils.print_info(f'Time taken: {end-start}')
# Because the temporary directory exists, when extracting the `output`
# directory from the container via `docker cp` this creates an `output`
# directory inside the directory
output_directory_name = os.path.join(output_directory_name, 'output')
if bless:
if actual_exit_status == expected_exit_status:
test_utils.print_info('Blessing output...')
shutil.copyfile(logs_file.name, expected_logs_file_name)
shutil.rmtree(expected_output_directory)
if len(os.listdir(output_directory_name)) != 0:
shutil.copytree(output_directory_name,
expected_output_directory,
symlinks=True)
test_utils.print_info('Test blessed')
else:
test_utils.print_error(
f'Was going to bless the test but it exited with status '
f'{actual_exit_status} (expected {actual_exit_status}')
return False
else:
if actual_exit_status != expected_exit_status:
test_utils.print_error(
f'Exited with a status code of {actual_exit_status}')
test_utils.print_error(
f'Expected a status code of {expected_exit_status}')
test_utils.print_error('Test failed')
return False
compare_logs_subp = test_utils.execute_command(
[
'git', '--no-pager', 'diff', '--no-index',
expected_logs_file_name, logs_file.name
],
fail_on_failure=False)
if compare_logs_subp.returncode != 0:
test_utils.print_error(
'Expected logs differ from the actual ones')
test_utils.print_error('Test failed')
return False
if os.path.isdir(expected_output_directory):
compare_output_subp = test_utils.execute_command(
[
'git', '--no-pager', 'diff', '--no-index',
expected_output_directory, output_directory_name
],
fail_on_failure=False)
if compare_output_subp.returncode != 0:
test_utils.print_error(
'Expected output differ from the actual one')
test_utils.print_error('Test failed')
return False
test_utils.print_info('Test passed')
return True
def __create_integration_instance(client, integration_name, integration_params, is_byoi):
# get configuration config (used for later rest api
configuration = __get_integration_config(client, integration_name)
if not configuration:
return None
module_configuration = configuration['configuration']
if not module_configuration:
module_configuration = []
instance_name = integration_name + '_test' + str(uuid.uuid4())
# define module instance
module_instance = {
'brand': configuration['name'],
'category': configuration['category'],
'configuration': configuration,
'data': [],
'enabled': "true",
'engine': '',
'id': '',
'isIntegrationScript': is_byoi,
'name': instance_name,
'passwordProtected': False,
'version': 0
}
# set module params
for param_conf in module_configuration:
if param_conf['display'] in integration_params or param_conf['name'] in integration_params:
# param defined in conf
key = param_conf['display'] if param_conf['display'] in integration_params else param_conf['name']
if key == 'credentials':
credentials = integration_params[key]
param_value = {
'credential': '',
'identifier': credentials['identifier'],
'password': credentials['password'],
'passwordChanged': False
}
else:
param_value = integration_params[key]
param_conf['value'] = param_value
param_conf['hasvalue'] = True
elif param_conf['defaultValue']:
# param is required - take default falue
param_conf['value'] = param_conf['defaultValue']
module_instance['data'].append(param_conf)
res = client.req('PUT', '/settings/integration', module_instance)
if res.status_code != 200:
print_error('create instance failed with status code ' + str(res.status_code))
print_error(pformat(res.json()))
return None
integration_config = res.json()
module_instance['id'] = integration_config['id']
# test integration
test_succeed = __test_integration_instance(client, module_instance)
if not test_succeed:
return
return module_instance['id']
def test_integration(client,
integrations,
playbook_id,
options=None,
is_mock_run=False):
options = options if options is not None else {}
# create integrations instances
module_instances = []
for integration in integrations:
integration_name = integration.get('name', None)
integration_params = integration.get('params', None)
is_byoi = integration.get('byoi', True)
if is_mock_run:
configure_proxy_unsecure(integration_params)
module_instance = __create_integration_instance(
client, integration_name, integration_params, is_byoi)
if module_instance is None:
print_error('Failed to create instance')
__delete_integrations_instances(client, module_instances)
return False, -1
module_instances.append(module_instance)
print('Create integration %s succeed' % (integration_name, ))
# create incident with playbook
incident, inc_id = __create_incident_with_playbook(
client, 'inc_%s' % (playbook_id, ), playbook_id)
if not incident:
return False, -1
investigation_id = incident['investigationId']
if investigation_id is None or len(investigation_id) == 0:
print_error('Failed to get investigation id of incident:' + incident)
return False, -1
timeout_amount = options[
'timeout'] if 'timeout' in options else DEFAULT_TIMEOUT
timeout = time.time() + timeout_amount
i = 1
# wait for playbook to finish run
while True:
# give playbook time to run
time.sleep(1)
# fetch status
playbook_state = __get_investigation_playbook_state(
client, investigation_id)
if playbook_state == PB_Status.COMPLETED:
break
if playbook_state == PB_Status.FAILED:
print_error(playbook_id + ' failed with error/s')
__print_investigation_error(client, playbook_id, investigation_id)
break
if time.time() > timeout:
print_error(playbook_id + ' failed on timeout')
break
if i % DEFAULT_INTERVAL == 0:
print 'loop no.' + str(
i / DEFAULT_INTERVAL) + ', playbook state is ' + playbook_state
i = i + 1
__disable_integrations_instances(client, module_instances)
test_pass = playbook_state == PB_Status.COMPLETED
if test_pass:
# delete incident
__delete_incident(client, incident)
# delete integration instance
__delete_integrations_instances(client, module_instances)
return test_pass, inc_id
def main():
options = options_handler()
username = options.user
password = options.password
server = options.server
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
if not (username and password and server):
print_error('You must provide server user & password arguments')
sys.exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code is not 200:
print_error("Login has failed with status code " +
str(res.status_code))
sys.exit(1)
with open(conf_path) as data_file:
conf = json.load(data_file)
secret_conf = None
if secret_conf_path:
with open(secret_conf_path) as data_file:
secret_conf = json.load(data_file)
tests = conf['tests']
secret_params = secret_conf['integrations'] if secret_conf else []
if not tests or len(tests) is 0:
print('no integrations are configured for test')
return
succeed_playbooks = []
failed_playbooks = []
for t in tests:
test_options = {
'timeout':
t['timeout'] if 'timeout' in t else conf.get('testTimeout', 30),
'interval':
conf.get('testInterval', 10)
}
playbook_id = t['playbookID']
integrations_conf = t.get('integrations', [])
if not isinstance(integrations_conf, list):
integrations_conf = [integrations_conf]
integrations = []
for integration in integrations_conf:
if type(integration) is dict:
# dict description
integrations.append({
'name': integration.get('name'),
'byoi': integration.get('byoi', True),
'params': {}
})
else:
# string description
integrations.append({
'name': integration,
'byoi': True,
'params': {}
})
for integration in integrations:
integration_params = [
item for item in secret_params
if item["name"] == integration['name']
]
if integration_params:
integration['params'] = integration_params[0].get('params', {})
test_message = 'playbook: ' + playbook_id
if integrations:
integrations_names = [
integration['name'] for integration in integrations
]
test_message = test_message + ' with integration(s): ' + ','.join(
integrations_names)
print '------ Test %s start ------' % (test_message, )
nightly_test = t.get('nightly', False)
skip_test = True if nightly_test and not is_nightly else False
if skip_test:
print 'Skip test'
else:
# run test
succeed = test_integration(c, integrations, playbook_id,
test_options)
# use results
if succeed:
print 'PASS: %s succeed' % (test_message, )
succeed_playbooks.append(playbook_id)
else:
print 'Failed: %s failed' % (test_message, )
failed_playbooks.append(playbook_id)
print '------ Test %s end ------' % (test_message, )
print_test_summary(succeed_playbooks, failed_playbooks)
if len(failed_playbooks):
sys.exit(1)
def main():
options = options_handler()
username = options.user
password = options.password
server = options.server
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
if not (username and password and server):
print_error('You must provide server user & password arguments')
sys.exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code is not 200:
print_error("Login has failed with status code " +
str(res.status_code))
sys.exit(1)
demisto_api_key = ''.join(
random.choice(string.ascii_letters + string.digits) for _ in range(32))
apikey_json = {'name': 'test_apikey', 'apikey': demisto_api_key}
c.req('POST', '/apikeys', apikey_json)
with open(conf_path) as data_file:
conf = json.load(data_file)
secret_conf = None
if secret_conf_path:
with open(secret_conf_path) as data_file:
secret_conf = json.load(data_file)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
skipped_integrations_conf = conf['skipped_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
with open(FILTER_CONF, 'r') as filter_file:
filterd_tests = filter_file.readlines()
filterd_tests = [line.strip('\n') for line in filterd_tests]
is_filter_configured = True if filterd_tests else False
if not tests or len(tests) is 0:
print('no integrations are configured for test')
return
succeed_playbooks = []
failed_playbooks = []
skipped_tests = []
for t in tests:
playbook_id = t['playbookID']
integrations_conf = t.get('integrations', [])
if playbook_id in skipped_tests_conf:
skipped_tests.append(playbook_id)
continue
if is_filter_configured and not is_nightly and playbook_id not in filterd_tests:
continue
test_options = {
'timeout':
t['timeout'] if 'timeout' in t else conf.get('testTimeout', 30),
'interval':
conf.get('testInterval', 10)
}
if not isinstance(integrations_conf, list):
integrations_conf = [integrations_conf]
integrations = []
has_skipped_integration = False
for integration in integrations_conf:
if type(integration) is dict:
if integration.get('name') in skipped_integrations_conf:
has_skipped_integration = True
break
# dict description
integrations.append({
'name': integration.get('name'),
'byoi': integration.get('byoi', True),
'params': {}
})
else:
if integration in skipped_integrations_conf:
has_skipped_integration = True
break
# string description
integrations.append({
'name': integration,
'byoi': True,
'params': {}
})
if has_skipped_integration:
continue
for integration in integrations:
integration_params = [
item for item in secret_params
if item["name"] == integration['name']
]
if integration_params:
integration['params'] = integration_params[0].get('params', {})
elif 'Demisto REST API' == integration['name']:
integration['params'] = {
'url': 'https://localhost',
'apikey': demisto_api_key,
'insecure': True,
}
test_message = 'playbook: ' + playbook_id
if integrations:
integrations_names = [
integration['name'] for integration in integrations
]
test_message = test_message + ' with integration(s): ' + ','.join(
integrations_names)
print '------ Test %s start ------' % (test_message, )
nightly_test = t.get('nightly', False)
skip_test = True if nightly_test and not is_nightly else False
if skip_test:
print 'Skip test'
else:
# run test
succeed = test_integration(c, integrations, playbook_id,
test_options)
# use results
if succeed:
print 'PASS: %s succeed' % (test_message, )
succeed_playbooks.append(playbook_id)
else:
print 'Failed: %s failed' % (test_message, )
failed_playbooks.append(playbook_id)
print '------ Test %s end ------' % (test_message, )
print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests)
os.remove(FILTER_CONF)
if len(failed_playbooks):
sys.exit(1)
