def install_packs_private(build: Build,
prints_manager: ParallelPrintsManager,
pack_ids: list = None) -> bool:
"""
Wrapper for the search and install packs function.
:param build: Build object containing the build settings.
:param prints_manager: PrintsManager object used for reporting status. Will be deprecated.
:param pack_ids: Optional, list of packs to install. List contains pack id and version requested.
:return: Boolean indicating if the installation was successful.
"""
pack_ids = pack_ids if pack_ids else build.pack_ids_to_install
installed_content_packs_successfully = True
for server in build.servers:
try:
flag = search_and_install_packs_and_their_dependencies_private(
build.test_pack_path, pack_ids, server.client)
if not flag:
raise Exception('Failed to search and install packs.')
except Exception as exc:
prints_manager.add_print_job(str(exc), print_error, 0)
prints_manager.execute_thread_prints(0)
installed_content_packs_successfully = False
return installed_content_packs_successfully
def test_instances(secret_conf_path, server, username, password):
integrations = get_integrations(secret_conf_path)
instance_ids = []
failed_integrations = []
integrations_counter = 0
prints_manager = ParallelPrintsManager(1)
content_installation_client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
install_new_content(content_installation_client, server)
for integration in integrations:
c = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
integrations_counter += 1
integration_name = integration.get('name')
integration_instance_name = integration.get('instance_name', '')
integration_params = integration.get('params')
devops_comments = integration.get('devops_comments')
product_description = integration.get('product_description', '')
is_byoi = integration.get('byoi', True)
has_integration = integration.get('has_integration', True)
validate_test = integration.get('validate_test', True)
if has_integration:
instance_id, failure_message, _ = __create_integration_instance(
c,
integration_name,
integration_instance_name,
integration_params,
is_byoi,
prints_manager,
validate_test=validate_test)
if failure_message == 'No configuration':
print_warning(
"Warning: skipping {} as it exists in content-test-conf conf.json but not "
"in content repo".format(integration_name))
continue
if not instance_id:
print_error(
'Failed to create instance of {} with message: {}'.format(
integration_name, failure_message))
failed_integrations.append(
"{} {} - devops comments: {}".format(
integration_name, product_description,
devops_comments))
else:
instance_ids.append(instance_id)
print('Create integration %s succeed' % (integration_name, ))
__delete_integrations_instances(c, instance_ids,
prints_manager)
prints_manager.execute_thread_prints(0)
return failed_integrations, integrations_counter
def search_and_install_packs_and_their_dependencies_private(
test_pack_path: str,
pack_ids: list,
client: demisto_client,
prints_manager: ParallelPrintsManager,
thread_index: int = 0):
""" Searches for the packs from the specified list, searches their dependencies, and then installs them.
Args:
test_pack_path (str): Path of where the test packs are located.
pack_ids (list): A list of the pack ids to search and install.
client (demisto_client): The client to connect to.
prints_manager (ParallelPrintsManager): A prints manager object.
thread_index (int): the thread index.
Returns (list, bool):
A list of the installed packs' ids, or an empty list if is_nightly == True.
A flag that indicates if the operation succeeded or not.
"""
host = client.api_client.configuration.host
msg = f'Starting to search and install packs in server: {host}'
prints_manager.add_print_job(msg, print_color, thread_index,
LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
install_packs_private(client, host, prints_manager, thread_index, pack_ids,
test_pack_path)
return SUCCESS_FLAG
def search_and_install_packs_and_their_dependencies_private(
test_pack_path: str,
pack_ids: list,
client: demisto_client,
prints_manager: ParallelPrintsManager,
thread_index: int = 0):
""" Searches for the packs from the specified list, searches their dependencies, and then installs them.
Args:
test_pack_path (str): Path of where the test packs are located.
pack_ids (list): A list of the pack ids to search and install.
client (demisto_client): The client to connect to.
prints_manager (ParallelPrintsManager): A prints manager object.
thread_index (int): the thread index.
Returns (list, bool):
A list of the installed packs' ids, or an empty list if is_nightly == True.
A flag that indicates if the operation succeeded or not.
"""
host = client.api_client.configuration.host
msg = f'Starting to search and install packs in server: {host}'
prints_manager.add_print_job(msg, print_color, thread_index,
LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
packs_to_install = [
] # we save all the packs we want to install, to avoid duplications
installation_request_body = [
] # the packs to install, in the request format
threads_list = []
lock = Lock()
for pack_id in pack_ids:
thread = Thread(target=search_pack_and_its_dependencies,
kwargs={
'client': client,
'prints_manager': prints_manager,
'pack_id': pack_id,
'packs_to_install': packs_to_install,
'installation_request_body':
installation_request_body,
'thread_index': thread_index,
'lock': lock
})
threads_list.append(thread)
run_threads_list(threads_list)
install_packs_private(client, host, prints_manager, thread_index, pack_ids,
test_pack_path)
return packs_to_install, SUCCESS_FLAG
def install_private_testing_pack(build: Build,
prints_manager: ParallelPrintsManager,
test_pack_zip_path: str):
"""
Creates and installs the test pack used in the private build. This pack contains the test
playbooks and test scripts that will be used for the tests.
:param build: Build object containing the build settings.
:param prints_manager: PrintsManager object used for reporting status. Will be deprecated.
:param test_pack_zip_path: Path to test_pack zip.
:return: No object is returned. nightly_install_packs will wait for the process to finish.
"""
threads_print_manager = ParallelPrintsManager(len(build.servers))
nightly_install_packs(build,
threads_print_manager,
install_method=upload_zipped_packs,
pack_path=test_pack_zip_path)
prints_manager.add_print_job('Sleeping for 45 seconds...',
print_warning,
0,
include_timestamp=True)
prints_manager.execute_thread_prints(0)
def upload_zipped_packs(client: demisto_client, host: str,
prints_manager: ParallelPrintsManager,
thread_index: int, pack_path: str):
""" Install packs from zip file.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
prints_manager (ParallelPrintsManager): Print manager object.
thread_index (int): the index (for prints_manager).
pack_path (str): path to pack zip.
"""
header_params = {'Content-Type': 'multipart/form-data'}
file_path = os.path.abspath(pack_path)
files = {'file': file_path}
message = 'Making "POST" request to server {} - to install all packs from file {}'.format(
host, pack_path)
prints_manager.add_print_job(message, print_color, thread_index,
LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
# make the pack installation request
try:
response_data, status_code, _ = client.api_client.call_api(
resource_path='/contentpacks/installed/upload',
method='POST',
header_params=header_params,
files=files)
if 200 <= status_code < 300:
message = 'All packs from {} were successfully installed!\n'.format(
pack_path)
prints_manager.add_print_job(message, print_color, thread_index,
LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
err_msg = 'Failed to install packs - with status code {}\n{}\n'.format(
status_code, message)
raise Exception(err_msg)
except Exception as e:
if e.__class__ == ApiException:
err_msg = 'The request to install packs has failed. Reason:\n{}\n'.format(
str(e.body))
else:
err_msg = 'The request to install packs has failed. Reason:\n{}\n'.format(
str(e))
prints_manager.add_print_job(err_msg, print_color, thread_index,
LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
sys.exit(1)
def install_new_content(client, server):
prints_manager = ParallelPrintsManager(1)
update_content_on_demisto_instance(client, server, prints_manager, 0)
prints_manager.execute_thread_prints(0)
def execute_testing(tests_settings: SettingsTester,
server_ip: str,
all_tests: set,
tests_data_keeper: DataKeeperTester,
prints_manager: ParallelPrintsManager,
thread_index: int = 0):
"""
Main function used to handle the testing process. Starts by turning off telemetry and disabling
any left over tests. Afterwards it will create a test queue object which then is used to run the
specific test scenario.
:param tests_settings: SettingsTester object which contains the test variables
:param server_ip: IP address of the server. Will be formatted before use.
:param all_tests: All tests currently in the test conf.
:param tests_data_keeper: Object containing all the test results. Used by report tests function.
:param prints_manager: PrintsManager object used in reporting. Will be deprecated.
:param thread_index: Integer indicating what thread the test is running on.
:return: No object is returned, just updates the tests_data_keep object.
"""
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion
start_message = "Executing tests with the server {} - and the server ip {}".format(
server, server_ip)
prints_manager.add_print_job(start_message, print, thread_index)
slack = tests_settings.slack
circle_ci = tests_settings.circleci
build_number = tests_settings.buildNumber
build_name = tests_settings.buildName
conf, secret_conf = load_conf_files(tests_settings.conf_path,
tests_settings.secret_conf_path)
demisto_api_key = tests_settings.api_key
demisto_user = secret_conf['username']
demisto_pass = secret_conf['userPassword']
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests = extract_filtered_tests()
if not tests or len(tests) == 0:
prints_manager.add_print_job('no integrations are configured for test',
print, thread_index)
prints_manager.execute_thread_prints(thread_index)
return
xsoar_client = demisto_client.configure(base_url=server,
username=demisto_user,
password=demisto_pass,
verify_ssl=False)
# turn off telemetry
turn_off_telemetry(xsoar_client)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
playbook_skipped_integration = set([])
disable_all_integrations(xsoar_client,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
# Private builds do not use mocking. Here we copy the mocked test list to the unmockable list.
private_tests = get_test_records_of_given_test_names(
tests_settings, all_tests)
try:
# first run the mock tests to avoid mockless side effects in container
prints_manager.add_print_job("\nRunning private tests", print,
thread_index)
executed_in_current_round, private_tests_queue = initialize_queue_and_executed_tests_set(
private_tests)
while not private_tests_queue.empty():
t = private_tests_queue.get()
executed_in_current_round = update_round_set_and_sleep_if_round_completed(
executed_in_current_round, prints_manager, t, thread_index)
run_private_test_scenario(tests_settings,
t,
default_test_timeout,
skipped_tests_conf,
nightly_integrations,
skipped_integrations_conf,
skipped_integration,
filtered_tests,
skipped_tests,
secret_params,
failed_playbooks,
playbook_skipped_integration,
succeed_playbooks,
slack,
circle_ci,
build_number,
server,
build_name,
server_numeric_version,
demisto_user,
demisto_pass,
demisto_api_key,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
except Exception as exc:
if exc.__class__ == ApiException:
error_message = exc.body
else:
error_message = f'~~ Thread {thread_index + 1} failed ~~\n{str(exc)}\n{traceback.format_exc()}'
prints_manager.add_print_job(error_message, print_error, thread_index)
prints_manager.execute_thread_prints(thread_index)
failed_playbooks.append(f'~~ Thread {thread_index + 1} failed ~~')
raise
finally:
tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks,
skipped_tests, skipped_integration,
unmockable_integrations)
def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
servers = determine_servers_urls(ami_env)
conf_path = options.conf
secret_conf_path = options.secret
prints_manager = ParallelPrintsManager(1)
server_numeric_version = get_server_numeric_version(ami_env)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
clients = []
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
clients.append(client)
testing_client = clients[0]
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(
is_nightly=False)
tests_for_iteration = tests
if run_all_tests:
# Use all tests for testing, leave 'tests_for_iteration' as is
pass
elif filter_configured and filtered_tests:
tests_for_iteration = [
test for test in tests
if test.get('playbookID', '') in filtered_tests
]
tests_for_iteration = filter_tests_with_incompatible_version(
tests_for_iteration, server_numeric_version, prints_manager)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_names, modified_integrations_names = get_new_and_modified_integrations(
git_sha1)
if new_integrations_names:
print_warning('New Integrations Since Last Release:\n{}\n'.format(
'\n'.join(new_integrations_names)))
if modified_integrations_names:
print_warning('Updated Integrations Since Last Release:\n{}\n'.format(
'\n'.join(modified_integrations_names)))
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
integrations = get_integrations_for_test(test,
skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
print_warning('All Integrations for test "{}":'.format(
test.get('playbookID')))
print_warning(integrations_names)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names,
modified_integrations_names)
integrations_msg = '\n'.join([
'"{}" - {}'.format(key, val)
for key, val in integration_to_status.items()
])
print_warning('{}\n'.format(integrations_msg))
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
new_ints_params_set = set_integration_params(new_integrations,
secret_params,
instance_names_conf)
ints_to_configure_params_set = set_integration_params(
integrations_to_configure, secret_params, instance_names_conf)
if not new_ints_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(new_integrations)))
if not ints_to_configure_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(integrations_to_configure)))
if not (new_ints_params_set and ints_to_configure_params_set):
continue
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
print_warning(
'Start of Instance Testing ("Test" button) prior to Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Pre-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(clients))
# For each server url we install content
for thread_index, (client, server_url) in enumerate(zip(clients, servers)):
t = Thread(target=update_content_on_demisto_instance,
kwargs={
'client': client,
'server': server_url,
'prints_manager': threads_prints_manager,
'thread_index': thread_index
})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
new_integration_module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
print_warning(
'Start of Instance Testing ("Test" button) after the Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Post-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
__disable_integrations_instances(client, all_module_instances,
prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails,
new_integrations_names)
if not success:
sys.exit(1)
def install_nightly_packs(client: demisto_client,
host: str,
prints_manager: ParallelPrintsManager,
thread_index: int,
packs_to_install: List,
request_timeout: int = 999999):
"""
Install content packs on nightly build.
We will catch the exception if pack fails to install and send the request to install packs again without the
corrupted pack.
Args:
client(demisto_client): The configured client to use.
host (str): The server URL.
prints_manager (ParallelPrintsManager): Print manager object.
thread_index (int): the thread index.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
Returns:
None: No data returned.
"""
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
message = 'Installing the following packs in server {}:\n{}'.format(
host, packs_to_install_str)
prints_manager.add_print_job(message,
print_color,
thread_index,
LOG_COLORS.GREEN,
include_timestamp=True)
prints_manager.execute_thread_prints(thread_index)
# make the pack installation request
all_packs_install_successfully = False
request_data = {'packs': packs_to_install, 'ignoreWarnings': True}
while not all_packs_install_successfully:
try:
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{
'ID': pack.get('id'),
'CurrentVersion': pack.get('currentVersion')
} for pack in ast.literal_eval(response_data)]
packs_message = f'The following packs were successfully installed:\n{packs_data}'
prints_manager.add_print_job(packs_message,
print_color,
thread_index,
LOG_COLORS.GREEN,
include_timestamp=True)
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
err_msg = f'Failed to install packs - with status code {status_code}\n{message}\n'
prints_manager.add_print_job(err_msg,
print_error,
thread_index,
include_timestamp=True)
raise Exception(err_msg)
break
except Exception as e:
err_msg = f'The request to install packs has failed. Reason:\n{str(e)}\n'
prints_manager.add_print_job(err_msg,
print_error,
thread_index,
include_timestamp=True)
all_packs_install_successfully = False
malformed_pack_id = find_malformed_pack_id(str(e))
if not malformed_pack_id:
break
# Remove the malformed pack from the pack to install list.
packs = [
pack for pack in packs_to_install
if pack['id'] not in malformed_pack_id
]
request_data = {'packs': packs, 'ignoreWarnings': True}
finally:
prints_manager.execute_thread_prints(thread_index)
def search_pack(client: demisto_client, prints_manager: ParallelPrintsManager,
pack_display_name: str, pack_id: str, thread_index: int,
lock: Lock) -> dict:
""" Make a pack search request.
Args:
client (demisto_client): The configured client to use.
prints_manager (ParallelPrintsManager): Print manager object.
pack_display_name (string): The pack display name.
pack_id (string): The pack ID.
thread_index (int): the thread index.
lock (Lock): A lock object.
Returns:
(dict): Returns the pack data if found, or empty dict otherwise.
"""
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path=f'/contentpacks/marketplace/{pack_id}',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
print_msg = f'Found pack "{pack_display_name}" by its ID "{pack_id}" in bucket!\n'
prints_manager.add_print_job(print_msg, print_color,
thread_index, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
return pack_data
else:
print_msg = f'Did not find pack "{pack_display_name}" by its ID "{pack_id}" in bucket.\n'
prints_manager.add_print_job(print_msg, print_color,
thread_index, LOG_COLORS.RED)
prints_manager.execute_thread_prints(thread_index)
raise Exception(print_msg)
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception as e:
err_msg = f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed. Reason:\n{str(e)}'
prints_manager.add_print_job(err_msg, print_color, thread_index,
LOG_COLORS.RED)
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
def get_pack_dependencies(client: demisto_client,
prints_manager: ParallelPrintsManager,
pack_data: dict, thread_index: int, lock: Lock):
""" Get the pack's required dependencies.
Args:
client (demisto_client): The configured client to use.
prints_manager (ParallelPrintsManager): A prints manager object.
pack_data (dict): Contains the pack ID and version.
thread_index (int): the thread index.
lock (Lock): A lock object.
Returns:
(list) The pack's dependencies.
"""
pack_id = pack_data['id']
try:
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/search/dependencies',
method='POST',
body=[pack_data],
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
dependencies_data = []
dependants_ids = [pack_id]
reseponse_data = ast.literal_eval(response_data).get(
'dependencies', [])
create_dependencies_data_structure(reseponse_data, dependants_ids,
dependencies_data,
dependants_ids)
dependencies_str = ', '.join(
[dep['id'] for dep in dependencies_data])
if dependencies_data:
message = 'Found the following dependencies for pack {}:\n{}\n'.format(
pack_id, dependencies_str)
prints_manager.add_print_job(message, print_color,
thread_index, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(thread_index)
return dependencies_data
if status_code == 400:
err_msg = f"Unable to find dependencies for {pack_id}."
prints_manager.add_print_job(err_msg, print_color, thread_index,
LOG_COLORS.RED)
prints_manager.execute_thread_prints(thread_index)
return []
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = 'Failed to get pack {} dependencies - with status code {}\n{}\n'.format(
pack_id, status_code, msg)
raise Exception(err_msg)
except Exception as e:
err_msg = 'The request to get pack {} dependencies has failed. Reason:\n{}\n'.format(
pack_id, str(e))
prints_manager.add_print_job(err_msg, print_color, thread_index,
LOG_COLORS.RED)
prints_manager.execute_thread_prints(thread_index)
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
def install_packs(client: demisto_client,
host: str,
prints_manager: ParallelPrintsManager,
thread_index: int,
packs_to_install: list,
request_timeout: int = 999999,
is_nightly: bool = False):
""" Make a packs installation request.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
prints_manager (ParallelPrintsManager): Print manager object.
thread_index (int): the thread index.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
is_nightly (bool): Is the build nightly or not.
"""
if is_nightly:
install_nightly_packs(client, host, prints_manager, thread_index,
packs_to_install)
request_data = {'packs': packs_to_install, 'ignoreWarnings': True}
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
message = 'Installing the following packs in server {}:\n{}'.format(
host, packs_to_install_str)
prints_manager.add_print_job(message,
print_color,
thread_index,
LOG_COLORS.GREEN,
include_timestamp=True)
prints_manager.execute_thread_prints(thread_index)
# make the pack installation request
try:
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{
'ID': pack.get('id'),
'CurrentVersion': pack.get('currentVersion')
} for pack in ast.literal_eval(response_data)]
packs_message = f'The following packs were successfully installed:\n{packs_data}'
prints_manager.add_print_job(packs_message,
print_color,
thread_index,
LOG_COLORS.GREEN,
include_timestamp=True)
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
err_msg = f'Failed to install packs - with status code {status_code}\n{message}\n'
prints_manager.add_print_job(err_msg,
print_error,
thread_index,
include_timestamp=True)
raise Exception(err_msg)
except Exception as e:
err_msg = f'The request to install packs has failed. Reason:\n{str(e)}\n'
prints_manager.add_print_job(err_msg,
print_error,
thread_index,
include_timestamp=True)
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
prints_manager.execute_thread_prints(thread_index)
def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
conf_path = options.conf
secret_conf_path = options.secret
branch_name = options.branch
ci_build_number = options.build_number
servers = determine_servers_urls(ami_env)
server_numeric_version = get_server_numeric_version(ami_env)
prints_manager = ParallelPrintsManager(1)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
for server in servers:
client = demisto_client.configure(base_url=server, username=username, password=password,
verify_ssl=False)
set_marketplace_gcp_bucket_for_build(client, prints_manager, branch_name, ci_build_number)
print('Restarting servers to apply GCS server config ...')
ssh_string = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {}@{} ' \
'"sudo systemctl restart demisto"'
try:
subprocess.check_output(
ssh_string.format('ec2-user', server.replace('https://', '')), shell=True)
except subprocess.CalledProcessError as exc:
print(exc.output)
print('Done restarting servers.')
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(is_nightly=options.is_nightly)
tests_for_iteration = tests
if run_all_tests:
# skip test button testing
skipped_instance_test_message = 'Not running instance tests when {} is turned on'.format(RUN_ALL_TESTS_FORMAT)
prints_manager.add_print_job(skipped_instance_test_message, print_warning, 0)
tests_for_iteration = []
elif filter_configured and filtered_tests:
tests_for_iteration = [test for test in tests if test.get('playbookID', '') in filtered_tests]
tests_for_iteration = filter_tests_with_incompatible_version(tests_for_iteration, server_numeric_version,
prints_manager)
prints_manager.execute_thread_prints(0)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_files, modified_integrations_files = get_new_and_modified_integration_files(git_sha1)
new_integrations_names, modified_integrations_names = [], []
installed_content_packs_successfully = True
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
# sleep for one minute before starting to search and install packs to ensure bucket is ready
prints_manager.add_print_job('Sleeping for 1 minute...', print_warning, 0)
prints_manager.execute_thread_prints(0)
sleep(60)
pack_ids = get_pack_ids_to_install()
# install content packs in every server
for server_url in servers:
try:
client = demisto_client.configure(base_url=server_url, username=username, password=password,
verify_ssl=False)
search_and_install_packs_and_their_dependencies(pack_ids, client, prints_manager, options.is_nightly)
except Exception as exc:
prints_manager.add_print_job(str(exc), print_error, 0)
prints_manager.execute_thread_prints(0)
installed_content_packs_successfully = False
if new_integrations_files:
new_integrations_names = get_integration_names_from_files(new_integrations_files)
new_integrations_names_message = \
'New Integrations Since Last Release:\n{}\n'.format('\n'.join(new_integrations_names))
prints_manager.add_print_job(new_integrations_names_message, print_warning, 0)
if modified_integrations_files:
modified_integrations_names = get_integration_names_from_files(modified_integrations_files)
modified_integrations_names_message = \
'Updated Integrations Since Last Release:\n{}\n'.format('\n'.join(modified_integrations_names))
prints_manager.add_print_job(modified_integrations_names_message, print_warning, 0)
prints_manager.execute_thread_prints(0)
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integrations = get_integrations_for_test(test, skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
prints_manager.add_print_job('All Integrations for test "{}":'.format(test.get('playbookID')), print_warning, 0)
prints_manager.add_print_job(integrations_names, print_warning, 0)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names
)
integrations_msg = '\n'.join(['"{}" - {}'.format(key, val) for key, val in integration_to_status.items()])
prints_manager.add_print_job('{}\n'.format(integrations_msg), print_warning, 0)
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf,
placeholders_map)
ints_to_configure_params_set = set_integration_params(integrations_to_configure, secret_params,
instance_names_conf, placeholders_map)
if not new_ints_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(new_integrations)), print_error, 0)
if not ints_to_configure_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(integrations_to_configure)),
print_error, 0)
if not (new_ints_params_set and ints_to_configure_params_set):
continue
prints_manager.execute_thread_prints(0)
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if module_instance:
module_instances.append(module_instance)
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
preupdate_success = set()
postupdate_success = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) prior to Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Pre-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
else:
preupdate_success.add((instance_name, integration_of_instance))
if LooseVersion(server_numeric_version) < LooseVersion('6.0.0'):
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(servers))
# For each server url we install content
for thread_index, server_url in enumerate(servers):
client = demisto_client.configure(base_url=server_url, username=username,
password=password, verify_ssl=False)
t = Thread(target=update_content_on_demisto_instance,
kwargs={'client': client, 'server': server_url, 'ami_name': ami_env,
'prints_manager': threads_prints_manager,
'thread_index': thread_index})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_integration_module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if new_integration_module_instance:
new_integration_module_instances.append(new_integration_module_instance)
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) after the Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Post-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
else:
postupdate_success.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False)
__disable_integrations_instances(client, all_module_instances, prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails, preupdate_success, postupdate_success,
new_integrations_names, prints_manager)
prints_manager.execute_thread_prints(0)
if not success or not installed_content_packs_successfully:
sys.exit(2)