def main():
""" Main function for iterating over existing packs folder in content repo and creating json of all
packs dependencies. The logic of pack dependency is identical to sdk find-dependencies command.
"""
install_logging('Calculate_Packs_Dependencies.log', include_process_name=True, logger=logging)
option = option_handler()
output_path = option.output_path
id_set_path = option.id_set_path
id_set = get_id_set(id_set_path)
pack_dependencies_result: dict = {}
logging.info("Selecting packs for dependencies calculation")
packs = select_packs_for_calculation()
calculate_all_packs_dependencies(pack_dependencies_result, id_set, packs)
logging.info(f"Number of created pack dependencies entries: {len(pack_dependencies_result.keys())}")
# finished iteration over pack folders
logging.success("Finished dependencies calculation")
with open(output_path, 'w') as pack_dependencies_file:
json.dump(pack_dependencies_result, pack_dependencies_file, indent=4)
logging.success(f"Created packs dependencies file at: {output_path}")
def search_and_install_packs_and_their_dependencies(pack_ids: list,
client: demisto_client):
""" Searches for the packs from the specified list, searches their dependencies, and then
installs them.
Args:
pack_ids (list): A list of the pack ids to search and install.
client (demisto_client): The client to connect to.
Returns (list, bool):
A list of the installed packs' ids, or an empty list if is_nightly == True.
A flag that indicates if the operation succeeded or not.
"""
host = client.api_client.configuration.host
logging.info(f'Starting to search and install packs in server: {host}')
packs_to_install: list = [] # we save all the packs we want to install, to avoid duplications
installation_request_body: list = [] # the packs to install, in the request format
threads_list = []
lock = Lock()
for pack_id in pack_ids:
thread = Thread(target=search_pack_and_its_dependencies,
kwargs={'client': client,
'pack_id': pack_id,
'packs_to_install': packs_to_install,
'installation_request_body': installation_request_body,
'lock': lock})
threads_list.append(thread)
run_threads_list(threads_list)
install_packs(client, host, installation_request_body)
return packs_to_install, SUCCESS_FLAG
def upload_zipped_packs(client: demisto_client,
host: str,
pack_path: str):
""" Install packs from zip file.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
pack_path (str): path to pack zip.
"""
header_params = {
'Content-Type': 'multipart/form-data'
}
file_path = os.path.abspath(pack_path)
files = {'file': file_path}
logging.info(f'Making "POST" request to server {host} - to install all packs from file {pack_path}')
# make the pack installation request
try:
response_data, status_code, _ = client.api_client.call_api(resource_path='/contentpacks/installed/upload',
method='POST',
header_params=header_params, files=files)
if 200 <= status_code < 300:
logging.info(f'All packs from file {pack_path} were successfully installed on server {host}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception:
logging.exception('The request to install packs has failed.')
sys.exit(1)
def uninstall_packs(client: demisto_client, pack_ids: list):
"""
Args:
client (demisto_client): The client to connect to.
pack_ids: packs ids to uninstall
Returns:
True if uninstalling succeeded False otherwise.
"""
body = {"IDs": pack_ids}
try:
logging.info("Attempting to uninstall all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/installed/delete',
method='POST',
body=body,
accept='application/json',
_request_timeout=None)
except Exception as e:
logging.exception(
f'The request to uninstall packs has failed. Additional info: {str(e)}'
)
return False
return True
def get_index_json_data(service_account: str, production_bucket_name: str, extract_path: str, storage_base_path: str) \
-> Tuple[dict, str]:
"""Retrieve the index.json file from production bucket.
Args:
service_account: Path to gcloud service account
production_bucket_name: Production bucket name
extract_path: Full path of folder to extract the index.zip to
storage_base_path: The base path in the bucket
Returns:
(Dict: content of the index.json, Str: path to index.json)
"""
logging.info('Downloading and extracting index.zip from the cloud')
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(production_bucket_name)
index_folder_path, _, _ = download_and_extract_index(
production_bucket, extract_path, storage_base_path)
logging.info("Retrieving the index file")
index_file_path = os.path.join(index_folder_path,
f"{GCPConfig.INDEX_NAME}.json")
index_data = load_json(index_file_path)
return index_data, index_file_path
def wait_for_uninstallation_to_complete(client: demisto_client,
retries: int = 30):
"""
Query if there are still installed packs, as it might take time to complete.
Args:
client (demisto_client): The client to connect to.
retries: Max number of sleep priods.
Returns: True if all packs were uninstalled successfully
"""
retry = 0
try:
installed_packs = get_all_installed_packs(client)
while len(installed_packs) > 1:
if retry > retries:
raise Exception(
'Waiting time for packs to be uninstalled has passed, there are still installed '
'packs. Aborting.')
logging.info(
f'The process of uninstalling all packs is not over! There are still {len(installed_packs)} '
f'packs installed. Sleeping for 10 seconds.')
sleep(10)
installed_packs = get_all_installed_packs(client)
retry = retry + 1
except Exception as e:
logging.exception(
f'Exception while waiting for the packs to be uninstalled. The error is {e}'
)
return False
return True
def calculate_single_pack_dependencies(pack: str, dependency_graph: object) -> Tuple[dict, list, str]:
"""
Calculates pack dependencies given a pack and a dependencies graph.
First is extract the dependencies subgraph of the given graph only using DFS algorithm with the pack as source.
Then, for all the dependencies of that pack it Replaces the 'mandatory_for_packs' key with a boolean key 'mandatory'
which indicates whether this dependency is mandatory for this pack or not.
Then using that subgraph we get the first-level dependencies and all-levels dependencies.
Args:
pack: The pack for which we need to calculate the dependencies
dependency_graph: The full dependencies graph
Returns:
first_level_dependencies: A dict of the form {'dependency_name': {'mandatory': < >, 'display_name': < >}}
all_level_dependencies: A list with all dependencies names
pack: The pack name
"""
install_logging('Calculate_Packs_Dependencies.log', include_process_name=True, logger=logging)
first_level_dependencies = {}
all_level_dependencies = []
try:
logging.info(f"Calculating {pack} pack dependencies.")
subgraph = PackDependencies.get_dependencies_subgraph_by_dfs(dependency_graph, pack)
for dependency_pack, additional_data in subgraph.nodes(data=True):
logging.debug(f'Iterating dependency {dependency_pack} for pack {pack}')
additional_data['mandatory'] = pack in additional_data['mandatory_for_packs']
del additional_data['mandatory_for_packs']
first_level_dependencies, all_level_dependencies = parse_for_pack_metadata(subgraph, pack)
except Exception:
logging.exception(f"Failed calculating {pack} pack dependencies")
raise
return first_level_dependencies, all_level_dependencies, pack
def get_pack_names(target_packs: str) -> set:
"""
Retrieves the paths of all relevant packs (that aren't ignored)
Args:
target_packs (str): csv packs names or `All` for all available packs in content.
Returns: The list of paths of the packs
"""
if target_packs.lower() == "all":
if os.path.exists(PACKS_FULL_PATH):
all_packs = {p for p in os.listdir(PACKS_FULL_PATH) if p not in IGNORED_FILES}
logging.info(f"Number of selected packs to upload is: {len(all_packs)}")
# return all available packs names
return all_packs
else:
logging.error(f"Folder {PACKS_FOLDER} was not found at the following path: {PACKS_FULL_PATH}")
sys.exit(1)
elif target_packs and isinstance(target_packs, str):
modified_packs = {p.strip() for p in target_packs.split(',') if p not in IGNORED_FILES}
logging.info(f"Number of selected packs to upload is: {len(modified_packs)}")
# return only packs from csv list
return modified_packs
else:
logging.error("Not correct usage of flag -p. Please check help section of upload packs script.")
sys.exit(1)
def update_round_set_and_sleep_if_round_completed(
executed_in_current_round: set, t: dict) -> set:
"""
Checks if the string representation of the current test configuration is already in
the executed_in_current_round set.
If it is- it means we have already executed this test and the we have reached a round and
there are tests that
were not able to be locked by this execution..
In that case we want to start a new round monitoring by emptying the
'executed_in_current_round' set and sleep
in order to let the tests be unlocked
Args:
executed_in_current_round: A set containing the string representation of all tests
configuration as they appear
in conf.json file that were already executed in the current round
t: test configuration as it appears in conf.json file
Returns:
A new executed_in_current_round set which contains only the current tests configuration if a
round was completed else it just adds the new test to the set.
"""
if str(t) in executed_in_current_round:
logging.info(
'all tests in the queue were executed, sleeping for 30 seconds to let locked tests get unlocked.'
)
executed_in_current_round = set()
time.sleep(30)
executed_in_current_round.add(str(t))
return executed_in_current_round
def manage_tests(tests_settings: SettingsTester):
"""
This function manages the execution of Demisto's tests.
Args:
tests_settings (SettingsTester): An object containing all the relevant data regarding how the
tests should be ran.
"""
tests_settings.serverNumericVersion = get_server_numeric_version(
tests_settings.serverVersion, tests_settings.is_local_run)
instances_ips = get_instances_ips_and_names(tests_settings)
tests_data_keeper = DataKeeperTester()
for ami_instance_name, ami_instance_ip in instances_ips:
if ami_instance_name == tests_settings.serverVersion:
logging.info(f"Starting private testing for {ami_instance_name}")
logging.info(
f"Starts tests with server url - https://{ami_instance_ip}")
all_tests = get_all_tests(tests_settings)
execute_testing(tests_settings, ami_instance_ip, all_tests,
tests_data_keeper)
sleep(8)
print_test_summary(tests_data_keeper,
tests_settings.isAMI,
logging_module=logging)
create_result_files(tests_data_keeper)
if tests_data_keeper.failed_playbooks:
tests_failed_msg = "Some tests have failed. Not destroying instances."
print(tests_failed_msg)
sys.exit(1)
def xsiam_configure_and_install_flow(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
logging.info('Retrieving the credentials for Cortex XSIAM server')
xsiam_machine = options.xsiam_machine
xsiam_servers = get_json_file(options.xsiam_servers_path)
api_key, server_numeric_version, base_url, xdr_auth_id = XSIAMBuild.get_xsiam_configuration(
xsiam_machine, xsiam_servers)
# Configure the Server
server = XSIAMServer(api_key, server_numeric_version, base_url,
xdr_auth_id, xsiam_machine)
XSIAMBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
# Acquire the server's host and install new uploaded content packs
install_packs_from_content_packs_to_install_path(
[server], options.pack_ids_to_install, server.name)
logging.success(
f'Finished installing all content packs in {xsiam_machine}')
def run_test_logic(tests_settings: Any, c: Any, failed_playbooks: list,
integrations: list, playbook_id: str,
succeed_playbooks: list, test_message: str,
test_options: dict, slack: Any, circle_ci: str,
build_number: str, server_url: str, demisto_user: str,
demisto_pass: str, build_name: str) -> bool:
"""
run_test_logic handles the testing of the integration by triggering check_integration. afterwards
it will check the status of the test and report success or add the failed test to the list of
failed integrations.
:param tests_settings: SettingsTester object which contains the test variables
:param c: Client for connecting to XSOAR via demisto-py
:param failed_playbooks: List of failed playbooks, additional failed playbooks will be added if
they failed.
:param integrations: List of integrations being tested.
:param playbook_id: ID of the test playbook being tested.
:param succeed_playbooks: List of playbooks which have passed tests.
:param test_message: Name of the playbook/integration being tested. This is reported back in the
build and used to print in the console the test being ran.
:param test_options: Options being passed to the test. PID, Docker Threshold, Timeout, etc.
:param slack: Slack client used for notifications.
:param circle_ci: CircleCI token. Used to get name of dev who triggered the build.
:param build_number: The build number of the CI run. Used in slack message.
:param server_url: The FQDN of the server tests are being ran on.
:param demisto_user: Username of the demisto user running the tests.
:param demisto_pass: Password of the demisto user running the tests.
:param build_name: Name of the build. (Nightly, etc.)
:return: Boolean indicating if the test was successful.
"""
status, inc_id = check_integration(c,
server_url,
demisto_user,
demisto_pass,
integrations,
playbook_id,
options=test_options)
if status == PB_Status.COMPLETED:
logging.success(f'PASS: {test_message} succeed')
succeed_playbooks.append(playbook_id)
elif status == PB_Status.NOT_SUPPORTED_VERSION:
logging.info(f'PASS: {test_message} skipped - not supported version')
succeed_playbooks.append(playbook_id)
else:
logging.error(f'Failed: {test_message} failed')
playbook_id_with_mock = playbook_id
playbook_id_with_mock += " (Mock Disabled)"
failed_playbooks.append(playbook_id_with_mock)
succeed = status in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION)
return succeed
def install_packs(client: demisto_client,
host: str,
packs_to_install: list,
request_timeout: int = 999999,
is_nightly: bool = False):
""" Make a packs installation request.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
is_nightly (bool): Is the build nightly or not.
"""
if is_nightly:
install_nightly_packs(client, host, packs_to_install)
return
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
logging.info(f'Installing packs on server {host}')
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs on server {host}:\n{packs_to_install_str}')
# make the pack installation request
try:
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for
pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception as e:
logging.exception(f'The request to install packs has failed. Additional info: {str(e)}')
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
return SUCCESS_FLAG
def validate_valid_packs_in_sections(landing_page_sections_json: dict,
valid_pack_names: set) -> None:
"""
Validates all packs in the sections of the file are valid packs according to the latest index.zip file
Args:
landing_page_sections_json: The content of the landingPage_sections.json file
valid_pack_names: A set containing all valid pack names from latest index.zip file and content repo
"""
logging.info(
'validating packs in sections appear in latest index.zip file')
for section_name, packs_in_section in landing_page_sections_json.items():
if section_name in {'description', 'sections'}:
continue
for pack_name in packs_in_section:
assert pack_name in valid_pack_names, f'Pack {pack_name} was not found in latest index.zip file, ' \
f'Make sure you uploaded the pack'
def acquire_dummy_index_lock(public_storage_bucket, dummy_index_lock_path):
total_seconds_waited = 0
while is_dummy_index_locked(public_storage_bucket, dummy_index_lock_path):
if total_seconds_waited >= MAX_SECONDS_TO_WAIT_FOR_LOCK:
logging.critical(
"Error: Failed too long to acquire lock, exceeded max wait time."
)
sys.exit(1)
if total_seconds_waited % 60 == 0:
# Printing a message every minute to keep the machine from dying due to no output
logging.info("Waiting to acquire lock.")
total_seconds_waited += 10
time.sleep(10)
lock_dummy_index(public_storage_bucket, dummy_index_lock_path)
def reset_base_pack_version(client: demisto_client):
"""
Resets base pack version to prod version.
Args:
client (demisto_client): The client to connect to.
"""
host = client.api_client.configuration.host.replace(
'https://api-', 'https://') # disable-secrets-detection
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/Base',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
logging.debug('Found Base pack in bucket!')
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
# install latest version of Base pack
logging.info(
f'updating base pack to version {result_object.get("currentVersion")}'
)
return install_packs(client, host, [pack_data], False)
else:
raise Exception('Did not find Base pack')
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for base pack, failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception:
logging.exception('Search request Base pack has failed.')
return False
def uninstall_all_packs(client: demisto_client, hostname):
""" Lists all installed packs and uninstalling them.
Args:
client (demisto_client): The client to connect to.
hostname (str): xsiam hostname
Returns (list, bool):
A flag that indicates if the operation succeeded or not.
"""
logging.info(
f'Starting to search and uninstall packs in server: {hostname}')
packs_to_uninstall: list = get_all_installed_packs(client)
if packs_to_uninstall:
return uninstall_packs(client, packs_to_uninstall)
logging.debug('Skipping packs uninstallation - nothing to uninstall')
return True
def get_all_packs_dependency_graph(id_set: dict, packs: list) -> Iterable:
"""
Gets a graph with dependencies for all packs
Args:
id_set: The content of id_set file
packs: The packs that should be part of the dependencies calculation
Returns:
A graph with all packs dependencies
"""
logging.info("Calculating pack dependencies.")
try:
dependency_graph = PackDependencies.build_all_dependencies_graph(packs, id_set=id_set, verbose=False)
return dependency_graph
except Exception:
logging.exception("Failed calculating dependencies graph")
sys.exit(2)
def validate_file_keys(landing_page_sections_json: dict) -> None:
"""
Validates that besides the 'description' and 'sections' keys - all keys in the file are sections names that appear
in the 'sections' part of the file.
Raises: Exception if the file has non allowed key.
Args:
landing_page_sections_json: The content of the landingPage_sections.json file
"""
logging.info('Validating file keys are valid sections')
allowed_keys = {'description', 'sections'}
allowed_keys.update(landing_page_sections_json['sections'])
not_allowed_key = [
key for key in landing_page_sections_json.keys()
if key not in allowed_keys
]
assert not not_allowed_key, f'Unsupported keys found: {not_allowed_key}, please add ' \
f'these keys under the "sections" key or remove them.'
def search_and_install_packs_and_their_dependencies_private(
test_pack_path: str, pack_ids: list, client: demisto_client):
""" Searches for the packs from the specified list, searches their dependencies, and then installs them.
Args:
test_pack_path (str): Path of where the test packs are located.
pack_ids (list): A list of the pack ids to search and install.
client (demisto_client): The client to connect to.
Returns (list, bool):
A list of the installed packs' ids, or an empty list if is_nightly == True.
A flag that indicates if the operation succeeded or not.
"""
host = client.api_client.configuration.host
logging.info(f'Starting to search and install packs in server: {host}')
install_packs_private(client, host, pack_ids, test_pack_path)
return SUCCESS_FLAG
def update_content(content_zip_path,
server=None,
username=None,
password=None,
client=None):
"""Update the content on a demisto instance with the content files in a zip file.
Args:
server (str): URL of the demisto server instance.
username (str): Username to login to the demisto instance.
password (str): Password of the associated username to login to the demisto instance.
content_zip_path (str): The path to the zip file containing content files.
client (demisto_client): The configured client to use.
"""
try:
# Configure Demisto Client and make request to upload content zip file
if not client:
client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
file_path = os.path.abspath(content_zip_path)
files = {'file': file_path}
header_params = {'Content-Type': 'multipart/form-data'}
logging.info(
f'Making "POST" request to server - "{server}" to upload the content zip file "{content_zip_path}"'
)
response_data, status_code, _ = client.api_client.call_api(
resource_path='/content/upload',
method='POST',
header_params=header_params,
files=files)
if status_code >= 300 or status_code < 200:
result_object = ast.literal_eval(response_data)
message = result_object['message']
raise Exception(
f"Upload has failed with status code {status_code}\n{message}")
success_msg = f'"{content_zip_path}" successfully uploaded to server "{server}"'
logging.success(success_msg)
except Exception:
logging.exception(
f'Failed to upload {content_zip_path} to server {server}')
def install_packs_from_content_packs_to_install_path(servers,
pack_ids_to_install_path,
hostname=''):
"""
Install pack_ids from "$ARTIFACTS_FOLDER/content_packs_to_install.txt" file, and packs dependencies.
Args:
hostname:
pack_ids_to_install_path: "$ARTIFACTS_FOLDER/content_packs_to_install.txt" path
servers: XSIAM or XSOAR Servers to install packs on it.
"""
pack_ids = Build.fetch_pack_ids_to_install(pack_ids_to_install_path)
for server in servers:
logging.info(
f'Starting to install all content packs in {hostname if hostname else server.internal_ip}'
)
_, success = search_and_install_packs_and_their_dependencies(
pack_ids, server.client, hostname)
if not success:
raise Exception(
'Failed to search and install packs and their dependencies.')
def slack_notifier(slack_token, secret_conf_path, server, user, password,
build_url, build_number):
logging.info("Starting Slack notifications about instances")
attachments, integrations_counter = get_attachments(
secret_conf_path, server, user, password, build_url)
sc = SlackClient(slack_token)
# Failing instances list
sc.api_call("chat.postMessage",
json={
'channel':
'dmst-content-lab',
'username':
'******',
'as_user':
'******',
'attachments':
attachments,
'text':
"You have {0} instances configurations".format(
integrations_counter)
})
# Failing instances file
sc.api_call(
"chat.postMessage",
json={
'channel':
'dmst-content-lab',
'username':
'******',
'as_user':
'******',
'text':
"Detailed list of failing instances could be found in the following link:\n"
"https://{}-60525392-gh.circle-artifacts.com/0/artifacts/failed_instances.txt"
.format(build_number)
})
def check_index_data(index_data: dict) -> bool:
"""Check index.json file inside the index.zip archive in the cloud.
Validate by running verify_pack on each pack.
Args:
index_data: Dictionary of the index.json contents.
Returns: True if all packs are valid, False otherwise.
"""
logging.info("Found index data in index file. Checking...")
logging.debug(f"Index data is:\n {pformat(index_data)}")
packs_list_exists = log_message_if_statement(
statement=(len(index_data.get("packs", [])) != 0),
error_message="Found 0 packs in index file."
"\nAborting the rest of the check.")
# If all packs are gone, return False
if not packs_list_exists:
return False
mandatory_pack_ids = load_json(MANDATORY_PREMIUM_PACKS_PATH).get(
"packs", [])
packs_are_valid = True
for pack in index_data["packs"]:
pack_is_good = verify_pack(pack)
if not pack_is_good:
packs_are_valid = False
if pack["id"] in mandatory_pack_ids:
mandatory_pack_ids.remove(pack["id"])
all_mandatory_packs_are_found = log_message_if_statement(
statement=(mandatory_pack_ids == []),
error_message=f"index json is missing some mandatory"
f" pack ids: {pformat(mandatory_pack_ids)}",
success_message="All premium mandatory pack ids were"
" found in the index.json file.")
return all([packs_are_valid, all_mandatory_packs_are_found])
def get_all_installed_packs(client: demisto_client):
"""
Args:
client (demisto_client): The client to connect to.
Returns:
list of installed python
"""
try:
logging.info("Attempting to fetch all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/metadata/installed',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
installed_packs = ast.literal_eval(response_data)
installed_packs_ids = [pack.get('id') for pack in installed_packs]
logging.success('Successfully fetched all installed packs.')
installed_packs_ids_str = ', '.join(installed_packs_ids)
logging.debug(
f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}'
)
if 'Base' in installed_packs_ids:
installed_packs_ids.remove('Base')
return installed_packs_ids
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(
f'Failed to fetch installed packs - with status code {status_code}\n{message}'
)
except Exception as e:
logging.exception(
f'The request to fetch installed packs has failed. Additional info: {str(e)}'
)
return None
def xsoar_configure_and_install_flow(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
# Get the host by the ami env
server_to_port_mapping, server_version = XSOARBuild.get_servers(
ami_env=options.ami_env)
logging.info('Retrieving the credentials for Cortex XSOAR server')
secret_conf_file = get_json(file_path=options.secret)
username: str = secret_conf_file.get('username')
password: str = secret_conf_file.get('userPassword')
servers = []
# Configure the Servers
for server_url, port in server_to_port_mapping.items():
server = XSOARServer(internal_ip=server_url,
port=port,
user_name=username,
password=password)
logging.info(f'Adding Marketplace configuration to {server_url}')
error_msg: str = 'Failed to set marketplace configuration.'
server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION,
error_msg=error_msg)
XSOARBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
servers.append(server)
install_packs_from_content_packs_to_install_path(
servers, options.pack_ids_to_install)
logging.success(
f'Finished installing all content packs in {[server.internal_ip for server in servers]}'
)
def main():
install_logging('cleanup_xsiam_instance.log', logger=logging)
# in xsiam we dont use demisto username
os.environ.pop('DEMISTO_USERNAME', None)
options = options_handler()
host = options.xsiam_machine
xsiam_servers = get_json_file(options.xsiam_servers_path)
api_key, base_url, xdr_auth_id = get_xsiam_configuration(
options.xsiam_machine, xsiam_servers)
logging.info(f'Starting cleanup for XSIAM server {host}')
client = demisto_client.configure(base_url=base_url,
verify_ssl=False,
api_key=api_key,
auth_id=xdr_auth_id)
success = reset_base_pack_version(client) and uninstall_all_packs(
client, host) and wait_for_uninstallation_to_complete(client)
if not success:
sys.exit(2)
logging.info('Uninstalling packs done.')
def run_test(tests_settings: SettingsTester, demisto_user: str,
demisto_pass: str, failed_playbooks: list, integrations: list,
playbook_id: str, succeed_playbooks: list, test_message: str,
test_options: dict, slack: str, circle_ci: str, build_number: str,
server_url: str, build_name: str) -> None:
"""
Wrapper for the run_test_logic function. Helps by indicating when the test is starting and ending.
:param tests_settings: SettingsTester object which contains the test variables
:param demisto_user: Username of the demisto user running the tests.
:param demisto_pass: Password of the demisto user running the tests.
:param failed_playbooks: List of failed playbooks, additional failed playbooks will be added if
they failed.
:param integrations: List of integrations being tested.
:param playbook_id: ID of the test playbook being tested.
:param succeed_playbooks: List of playbooks which have passed tests.
:param test_message: Name of the playbook/integration being tested. This is reported back in the
build and used to print in the console the test being ran.
:param test_options: Options being passed to the test. PID, Docker Threshold, Timeout, etc.
:param slack: Slack client used for notifications.
:param circle_ci: CircleCI token. Used to get name of dev who triggered the build.
:param build_number: The build number of the CI run. Used in slack message.
:param server_url: The FQDN of the server tests are being ran on.
:param build_name: Name of the build. (Nightly, etc.)
:return: No object is returned.
"""
start_message = f'------ Test {test_message} start ------'
client = demisto_client.configure(base_url=server_url,
username=demisto_user,
password=demisto_pass,
verify_ssl=False)
logging.info(f'{start_message} (Private Build Test)')
run_test_logic(tests_settings, client, failed_playbooks, integrations,
playbook_id, succeed_playbooks, test_message, test_options,
slack, circle_ci, build_number, server_url, demisto_user,
demisto_pass, build_name)
logging.info(f'------ Test {test_message} end ------\n')
def xsoar_configure_and_install_all_packs(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
# Get the host by the ami env
server_to_port_mapping, server_version = XSOARBuild.get_servers(
ami_env=options.ami_env)
logging.info('Retrieving the credentials for Cortex XSOAR server')
secret_conf_file = get_json(file_path=options.secret)
username: str = secret_conf_file.get('username')
password: str = secret_conf_file.get('userPassword')
# Configure the Servers
for server_url, port in server_to_port_mapping.items():
server = XSOARServer(internal_ip=server_url,
port=port,
user_name=username,
password=password)
logging.info(f'Adding Marketplace configuration to {server_url}')
error_msg: str = 'Failed to set marketplace configuration.'
server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION,
error_msg=error_msg)
XSOARBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
# Acquire the server's host and install all content packs (one threaded execution)
logging.info(f'Starting to install all content packs in {server_url}')
server_host: str = server.client.api_client.configuration.host
success_flag = install_all_content_packs_from_build_bucket(
client=server.client,
host=server_host,
server_version=server_version,
bucket_packs_root_path=GCPConfig.
BUILD_BUCKET_PACKS_ROOT_PATH.format(branch=branch_name,
build=build_number,
marketplace='xsoar'),
service_account=options.service_account,
extract_destination_path=options.extract_path)
if success_flag:
logging.success(
f'Finished installing all content packs in {server_url}')
else:
logging.error('Failed to install all packs.')
sys.exit(1)
def main():
install_logging("GetPrivateBuildStatus.log", logger=logging)
if not os.path.isfile(PRIVATE_REPO_WORKFLOW_ID_FILE):
logging.info('Build private repo skipped')
sys.exit(0)
# gets workflow id from the file
with open(PRIVATE_REPO_WORKFLOW_ID_FILE, 'r') as f:
workflow_id = f.read()
# get github_token parameter
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--github-token', help='Github token')
args = arg_parser.parse_args()
github_token = args.github_token
# gets the workflow status
status, conclusion, step = get_workflow_status(github_token, workflow_id)
# initialize timer
start = time.time()
elapsed: float = 0
# polling the workflow status while is in progress
while status in ['queued', 'in_progress'
] and elapsed < GET_WORKFLOWS_TIMEOUT_THRESHOLD:
logging.info(
f'Workflow {workflow_id} status is {status}, current step: {step}')
time.sleep(60)
status, conclusion, step = get_workflow_status(github_token,
workflow_id)
elapsed = time.time() - start
if elapsed >= GET_WORKFLOWS_TIMEOUT_THRESHOLD:
logging.critical(
f'Timeout reached while waiting for private content build to complete, build url:'
f' {WORKFLOW_HTML_URL}/{workflow_id}')
sys.exit(1)
logging.info(f'Workflow {workflow_id} conclusion is {conclusion}')
if conclusion != 'success':
logging.critical(
f'Private repo build failed, build url: {WORKFLOW_HTML_URL}/{workflow_id}'
)
sys.exit(1)
logging.success('Build private repo finished successfully')
sys.exit(0)
