def copy_id_set(production_bucket: Bucket, build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the id_set.json artifact from the build bucket to the production bucket.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied from.
storage_base_path (str): the path to upload the id_set.json to.
build_bucket_base_path (str): the path in the build bucket of the id_set.json.
"""
build_id_set_path = os.path.join(os.path.dirname(build_bucket_base_path), 'id_set.json')
build_id_set_blob = build_bucket.blob(build_id_set_path)
if not build_id_set_blob.exists():
logging.error(f"id_set.json file does not exists in build bucket in path: {build_id_set_path}")
sys.exit(1)
prod_id_set_path = os.path.join(os.path.dirname(storage_base_path), 'id_set.json')
try:
copied_blob = build_bucket.copy_blob(
blob=build_id_set_blob, destination_bucket=production_bucket, new_name=prod_id_set_path
)
if not copied_blob.exists():
logging.error(f"Failed to upload id_set.json to {prod_id_set_path}")
sys.exit(1)
else:
logging.success("Finished uploading id_set.json to storage.")
except Exception as e:
logging.exception(f"Failed copying ID Set. Additional Info: {str(e)}")
sys.exit(1)
def wait_for_uninstallation_to_complete(client: demisto_client,
retries: int = 30):
"""
Query if there are still installed packs, as it might take time to complete.
Args:
client (demisto_client): The client to connect to.
retries: Max number of sleep priods.
Returns: True if all packs were uninstalled successfully
"""
retry = 0
try:
installed_packs = get_all_installed_packs(client)
while len(installed_packs) > 1:
if retry > retries:
raise Exception(
'Waiting time for packs to be uninstalled has passed, there are still installed '
'packs. Aborting.')
logging.info(
f'The process of uninstalling all packs is not over! There are still {len(installed_packs)} '
f'packs installed. Sleeping for 10 seconds.')
sleep(10)
installed_packs = get_all_installed_packs(client)
retry = retry + 1
except Exception as e:
logging.exception(
f'Exception while waiting for the packs to be uninstalled. The error is {e}'
)
return False
return True
def uninstall_packs(client: demisto_client, pack_ids: list):
"""
Args:
client (demisto_client): The client to connect to.
pack_ids: packs ids to uninstall
Returns:
True if uninstalling succeeded False otherwise.
"""
body = {"IDs": pack_ids}
try:
logging.info("Attempting to uninstall all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/installed/delete',
method='POST',
body=body,
accept='application/json',
_request_timeout=None)
except Exception as e:
logging.exception(
f'The request to uninstall packs has failed. Additional info: {str(e)}'
)
return False
return True
def upload_zipped_packs(client: demisto_client,
host: str,
pack_path: str):
""" Install packs from zip file.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
pack_path (str): path to pack zip.
"""
header_params = {
'Content-Type': 'multipart/form-data'
}
file_path = os.path.abspath(pack_path)
files = {'file': file_path}
logging.info(f'Making "POST" request to server {host} - to install all packs from file {pack_path}')
# make the pack installation request
try:
response_data, status_code, _ = client.api_client.call_api(resource_path='/contentpacks/installed/upload',
method='POST',
header_params=header_params, files=files)
if 200 <= status_code < 300:
logging.info(f'All packs from file {pack_path} were successfully installed on server {host}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception:
logging.exception('The request to install packs has failed.')
sys.exit(1)
def calculate_single_pack_dependencies(pack: str, dependency_graph: object) -> Tuple[dict, list, str]:
"""
Calculates pack dependencies given a pack and a dependencies graph.
First is extract the dependencies subgraph of the given graph only using DFS algorithm with the pack as source.
Then, for all the dependencies of that pack it Replaces the 'mandatory_for_packs' key with a boolean key 'mandatory'
which indicates whether this dependency is mandatory for this pack or not.
Then using that subgraph we get the first-level dependencies and all-levels dependencies.
Args:
pack: The pack for which we need to calculate the dependencies
dependency_graph: The full dependencies graph
Returns:
first_level_dependencies: A dict of the form {'dependency_name': {'mandatory': < >, 'display_name': < >}}
all_level_dependencies: A list with all dependencies names
pack: The pack name
"""
install_logging('Calculate_Packs_Dependencies.log', include_process_name=True, logger=logging)
first_level_dependencies = {}
all_level_dependencies = []
try:
logging.info(f"Calculating {pack} pack dependencies.")
subgraph = PackDependencies.get_dependencies_subgraph_by_dfs(dependency_graph, pack)
for dependency_pack, additional_data in subgraph.nodes(data=True):
logging.debug(f'Iterating dependency {dependency_pack} for pack {pack}')
additional_data['mandatory'] = pack in additional_data['mandatory_for_packs']
del additional_data['mandatory_for_packs']
first_level_dependencies, all_level_dependencies = parse_for_pack_metadata(subgraph, pack)
except Exception:
logging.exception(f"Failed calculating {pack} pack dependencies")
raise
return first_level_dependencies, all_level_dependencies, pack
def lock_and_unlock_dummy_index(public_storage_bucket, dummy_index_lock_path):
try:
acquire_dummy_index_lock(public_storage_bucket, dummy_index_lock_path)
yield
except Exception:
logging.exception("Error in dummy index lock context manager.")
finally:
release_dummy_index_lock(public_storage_bucket, dummy_index_lock_path)
def search_pack(client: demisto_client, pack_display_name: str, pack_id: str,
lock: Lock) -> dict:
""" Make a pack search request.
Args:
client (demisto_client): The configured client to use.
pack_display_name (string): The pack display name.
pack_id (string): The pack ID.
lock (Lock): A lock object.
Returns:
(dict): Returns the pack data if found, or empty dict otherwise.
"""
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path=f'/contentpacks/marketplace/{pack_id}',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
logging.debug(
f'Found pack "{pack_display_name}" by its ID "{pack_id}" in bucket!'
)
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
return pack_data
else:
raise Exception(
f'Did not find pack "{pack_display_name}" by its ID "{pack_id}" in bucket.'
)
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception:
logging.exception(
f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed.'
)
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
return {}
def copy_index(index_folder_path: str, build_index_blob: Blob, build_index_generation: str, production_bucket: Bucket,
build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the build bucket index to the production bucket index path.
Args:
index_folder_path (str): index folder full path.
build_index_blob (Blob): google cloud storage object that represents build index.zip blob.
build_index_generation (str): downloaded build index generation.
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied from.
storage_base_path (str): the path to upload the index to.
build_bucket_base_path (str): the path in the build bucket of the index.
"""
try:
build_index_blob.reload()
build_current_index_generation = build_index_blob.generation
# disabling caching for prod index blob
prod_index_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.zip")
prod_index_blob = production_bucket.blob(prod_index_storage_path)
prod_index_blob.cache_control = "no-cache,max-age=0"
prod_index_json_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.json")
prod_index_json_blob = production_bucket.blob(prod_index_json_storage_path)
prod_index_json_blob.cache_control = "no-cache,max-age=0"
if build_current_index_generation == build_index_generation:
copied_index = build_bucket.copy_blob(
blob=build_index_blob, destination_bucket=production_bucket, new_name=prod_index_storage_path
)
if copied_index.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage.")
else:
logging.error("Failed copying index.zip from build index - blob does not exist.")
sys.exit(1)
copied_index_json_blob = build_bucket.blob(
os.path.join(build_bucket_base_path, f"{GCPConfig.INDEX_NAME}.json")
)
copied_index_json = build_bucket.copy_blob(
blob=copied_index_json_blob, destination_bucket=production_bucket, new_name=prod_index_json_storage_path
)
if copied_index_json.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.json to storage.")
else:
logging.error("Failed copying index.json from build index - blob does not exist.")
sys.exit(1)
else:
logging.error(f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation")
logging.error(f"Downloaded build index generation: {build_index_generation}")
logging.error(f"Current build index generation: {build_current_index_generation}")
sys.exit(1)
except Exception as e:
logging.exception(f"Failed copying {GCPConfig.INDEX_NAME}. Additional Info: {str(e)}")
sys.exit(1)
finally:
shutil.rmtree(index_folder_path)
def get_pack_dependencies(client: demisto_client, pack_data: dict, lock: Lock):
""" Get the pack's required dependencies.
Args:
client (demisto_client): The configured client to use.
pack_data (dict): Contains the pack ID and version.
lock (Lock): A lock object.
Returns:
(list) The pack's dependencies.
"""
pack_id = pack_data['id']
logging.debug(f'Getting dependencies for pack {pack_id}')
try:
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/search/dependencies',
method='POST',
body=[pack_data],
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
dependencies_data: list = []
dependants_ids = [pack_id]
reseponse_data = ast.literal_eval(response_data).get(
'dependencies', [])
create_dependencies_data_structure(reseponse_data, dependants_ids,
dependencies_data,
dependants_ids)
dependencies_str = ', '.join(
[dep['id'] for dep in dependencies_data])
if dependencies_data:
logging.debug(
f'Found the following dependencies for pack {pack_id}: {dependencies_str}'
)
return dependencies_data
if status_code == 400:
logging.error(f'Unable to find dependencies for {pack_id}.')
return []
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
raise Exception(
f'Failed to get pack {pack_id} dependencies - with status code {status_code}\n{msg}\n'
)
except Exception:
logging.exception(
f'The request to get pack {pack_id} dependencies has failed.')
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
def install_packs(client: demisto_client,
host: str,
packs_to_install: list,
request_timeout: int = 999999,
is_nightly: bool = False):
""" Make a packs installation request.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
is_nightly (bool): Is the build nightly or not.
"""
if is_nightly:
install_nightly_packs(client, host, packs_to_install)
return
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
logging.info(f'Installing packs on server {host}')
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs on server {host}:\n{packs_to_install_str}')
# make the pack installation request
try:
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for
pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception as e:
logging.exception(f'The request to install packs has failed. Additional info: {str(e)}')
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
return SUCCESS_FLAG
def upload_core_packs_config(production_bucket: Bucket, build_number: str, extract_destination_path: str,
build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
"""Uploads the corepacks.json file to the target bucket. This files contains all of the server's core packs, under
the key corepacks, and specifies which core packs should be upgraded upon XSOAR upgrade, under the key upgradeCorePacks.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is uploaded.
build_number (str): CircleCI build number.
extract_destination_path (str): Full path of folder to extract the corepacks file
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is downloaded from.
storage_base_path (str): the path to upload the corepacks.json to.
build_bucket_base_path (str): the path in the build bucket of the corepacks.json.
"""
# download the corepacks.json stored in the build bucket to temp dir
build_corepacks_file_path = os.path.join(build_bucket_base_path, GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob = build_bucket.blob(build_corepacks_file_path)
if not build_corepacks_blob.exists():
logging.critical(f"{GCPConfig.CORE_PACK_FILE_NAME} is missing in {build_bucket.name} bucket, exiting...")
sys.exit(1)
temp_corepacks_file_path = os.path.join(extract_destination_path, GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob.download_to_filename(temp_corepacks_file_path)
corepacks_file = load_json(temp_corepacks_file_path)
# change the storage paths to the prod bucket
corepacks_list = corepacks_file.get('corePacks', [])
try:
corepacks_list = [os.path.join(GCPConfig.GCS_PUBLIC_URL, production_bucket.name, storage_base_path,
LATEST_ZIP_REGEX.findall(corepack_path)[0]) for corepack_path in corepacks_list]
except IndexError:
corepacks_list_str = '\n'.join(corepacks_list)
logging.exception(f"GCS paths in build bucket corepacks.json file are not of format: "
f"{GCPConfig.GCS_PUBLIC_URL}/<BUCKET_NAME>/.../content/packs/...\n"
f"List of build bucket corepacks paths:\n{corepacks_list_str}")
sys.exit(1)
# construct core pack data with public gcs urls
core_packs_data = {
'corePacks': corepacks_list,
'upgradeCorePacks': corepacks_file.get('upgradeCorePacks', []),
'buildNumber': build_number
}
# upload core pack json file to gcs
prod_corepacks_file_path = os.path.join(storage_base_path, GCPConfig.CORE_PACK_FILE_NAME)
prod_corepacks_blob = production_bucket.blob(prod_corepacks_file_path)
prod_corepacks_blob.upload_from_string(json.dumps(core_packs_data, indent=4))
logging.success(f"Finished uploading {GCPConfig.CORE_PACK_FILE_NAME} to storage.")
def ProcessPoolHandler() -> ProcessPool:
""" Process pool Handler which terminate all processes in case of Exception.
Yields:
ProcessPool: Pebble process pool.
"""
with ProcessPool(max_workers=3) as pool:
try:
yield pool
except Exception:
logging.exception("Gracefully release all resources due to Error...")
raise
finally:
pool.close()
pool.join()
def wait_futures_complete(futures: List[ProcessFuture], done_fn: Callable):
"""Wait for all futures to complete, Raise exception if occurred.
Args:
futures: futures to wait for.
done_fn: Function to run on result.
Raises:
Exception: Raise caught exception for further cleanups.
"""
for future in as_completed(futures):
try:
result = future.result()
done_fn(result)
except Exception as e:
logging.exception(e)
raise
def reset_base_pack_version(client: demisto_client):
"""
Resets base pack version to prod version.
Args:
client (demisto_client): The client to connect to.
"""
host = client.api_client.configuration.host.replace(
'https://api-', 'https://') # disable-secrets-detection
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/Base',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
logging.debug('Found Base pack in bucket!')
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
# install latest version of Base pack
logging.info(
f'updating base pack to version {result_object.get("currentVersion")}'
)
return install_packs(client, host, [pack_data], False)
else:
raise Exception('Did not find Base pack')
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for base pack, failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception:
logging.exception('Search request Base pack has failed.')
return False
def get_all_packs_dependency_graph(id_set: dict, packs: list) -> Iterable:
"""
Gets a graph with dependencies for all packs
Args:
id_set: The content of id_set file
packs: The packs that should be part of the dependencies calculation
Returns:
A graph with all packs dependencies
"""
logging.info("Calculating pack dependencies.")
try:
dependency_graph = PackDependencies.build_all_dependencies_graph(packs, id_set=id_set, verbose=False)
return dependency_graph
except Exception:
logging.exception("Failed calculating dependencies graph")
sys.exit(2)
def update_content(content_zip_path,
server=None,
username=None,
password=None,
client=None):
"""Update the content on a demisto instance with the content files in a zip file.
Args:
server (str): URL of the demisto server instance.
username (str): Username to login to the demisto instance.
password (str): Password of the associated username to login to the demisto instance.
content_zip_path (str): The path to the zip file containing content files.
client (demisto_client): The configured client to use.
"""
try:
# Configure Demisto Client and make request to upload content zip file
if not client:
client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
file_path = os.path.abspath(content_zip_path)
files = {'file': file_path}
header_params = {'Content-Type': 'multipart/form-data'}
logging.info(
f'Making "POST" request to server - "{server}" to upload the content zip file "{content_zip_path}"'
)
response_data, status_code, _ = client.api_client.call_api(
resource_path='/content/upload',
method='POST',
header_params=header_params,
files=files)
if status_code >= 300 or status_code < 200:
result_object = ast.literal_eval(response_data)
message = result_object['message']
raise Exception(
f"Upload has failed with status code {status_code}\n{message}")
success_msg = f'"{content_zip_path}" successfully uploaded to server "{server}"'
logging.success(success_msg)
except Exception:
logging.exception(
f'Failed to upload {content_zip_path} to server {server}')
def add_pack_metadata_results(results: Tuple) -> None:
"""
This is a callback that should be called once the result of the future is ready.
The results include: first_level_dependencies, all_level_dependencies, pack_name
Using these results we write the dependencies
"""
try:
first_level_dependencies, all_level_dependencies, pack_name = results
logging.debug(f'Got dependencies for pack {pack_name}\n: {pformat(all_level_dependencies)}')
pack_dependencies_result[pack_name] = {
"dependencies": first_level_dependencies,
"displayedImages": list(first_level_dependencies.keys()),
"allLevelDependencies": all_level_dependencies,
"path": os.path.join(PACKS_FOLDER, pack_name),
"fullPath": os.path.abspath(os.path.join(PACKS_FOLDER, pack_name))
}
except Exception:
logging.exception('Failed to collect pack dependencies results')
raise
def upload_modified_index(public_index_folder_path, extract_destination_path,
public_ci_dummy_index_blob, build_number,
private_packs):
"""Upload updated index zip to cloud storage.
Args:
public_index_folder_path (str): public index folder full path.
extract_destination_path (str): extract folder full path.
public_ci_dummy_index_blob (Blob): google cloud storage object that represents the dummy index.zip blob.
build_number (str): circleCI build number, used as an index revision.
private_packs (list): List of private packs and their price.
"""
with open(os.path.join(public_index_folder_path, "index.json"),
"w+") as index_file:
for private_pack in private_packs:
private_pack['price'] = 0
index = {
'revision': build_number,
'modified': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
'packs': private_packs
}
json.dump(index, index_file, indent=4)
index_zip_name = os.path.basename(public_index_folder_path)
index_zip_path = shutil.make_archive(base_name=public_index_folder_path,
format="zip",
root_dir=extract_destination_path,
base_dir=index_zip_name)
try:
public_ci_dummy_index_blob.reload()
public_ci_dummy_index_blob.cache_control = "no-cache,max-age=0" # disabling caching for index blob
public_ci_dummy_index_blob.upload_from_filename(index_zip_path)
logging.success("Finished uploading index.zip to storage.")
except Exception:
logging.exception(
"Failed in uploading index. Mismatch in index file generation.")
sys.exit(1)
finally:
shutil.rmtree(public_index_folder_path)
def get_all_installed_packs(client: demisto_client):
"""
Args:
client (demisto_client): The client to connect to.
Returns:
list of installed python
"""
try:
logging.info("Attempting to fetch all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/metadata/installed',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
installed_packs = ast.literal_eval(response_data)
installed_packs_ids = [pack.get('id') for pack in installed_packs]
logging.success('Successfully fetched all installed packs.')
installed_packs_ids_str = ', '.join(installed_packs_ids)
logging.debug(
f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}'
)
if 'Base' in installed_packs_ids:
installed_packs_ids.remove('Base')
return installed_packs_ids
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(
f'Failed to fetch installed packs - with status code {status_code}\n{message}'
)
except Exception as e:
logging.exception(
f'The request to fetch installed packs has failed. Additional info: {str(e)}'
)
return None
def install_nightly_packs(client: demisto_client,
host: str,
packs_to_install: List,
request_timeout: int = 999999):
"""
Install content packs on nightly build.
We will catch the exception if pack fails to install and send the request to install packs again without the
corrupted pack.
Args:
client(demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
Returns:
None: No data returned.
"""
logging.info(f'Installing packs on server {host}')
# make the pack installation request
all_packs_install_successfully = False
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
while not all_packs_install_successfully:
try:
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs in server {host}:\n{packs_to_install_str}')
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs on server {host}- with status code {status_code}\n{message}\n')
break
except Exception as e:
all_packs_install_successfully = False
malformed_pack_id = find_malformed_pack_id(str(e))
if not malformed_pack_id:
logging.exception('The request to install packs has failed')
raise
pack_ids_to_install = {pack['id'] for pack in packs_to_install}
malformed_pack_id = malformed_pack_id[0]
if malformed_pack_id not in pack_ids_to_install:
logging.exception(
f'The pack {malformed_pack_id} has failed to install even though it was not in the installation list')
raise
logging.warning(f'The request to install packs on server {host} has failed, retrying without {malformed_pack_id}')
# Remove the malformed pack from the pack to install list.
packs_to_install = [pack for pack in packs_to_install if pack['id'] not in malformed_pack_id]
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
def test_instances(secret_conf_path, server, username, password):
integrations = get_integrations(secret_conf_path)
instance_ids = []
failed_integrations = []
integrations_counter = 0
content_installation_client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
install_new_content(content_installation_client, server)
for integration in integrations:
c = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
integrations_counter += 1
integration_name = integration.get('name')
integration_instance_name = integration.get('instance_name', '')
integration_params = integration.get('params')
devops_comments = integration.get('devops_comments')
product_description = integration.get('product_description', '')
is_byoi = integration.get('byoi', True)
has_integration = integration.get('has_integration', True)
validate_test = integration.get('validate_test', True)
if has_integration:
try:
instance_id, failure_message = __create_integration_instance(
server,
username,
password,
integration_name,
integration_instance_name,
integration_params,
is_byoi,
validate_test=validate_test)
except Exception:
logging.exception(
f'Failed to configure integration with name {integration_name}'
)
failed_integrations.append(
f"{integration_name} {product_description} - devops comments: {devops_comments}"
)
continue
if failure_message == 'No configuration':
logging.warning(
f"skipping {integration_name} as it exists in content-test-conf conf.json but not in content repo"
)
continue
if not instance_id:
logging.error(
f'Failed to create instance of {integration_name} with message: {failure_message}'
)
failed_integrations.append(
"{} {} - devops comments: {}".format(
integration_name, product_description,
devops_comments))
else:
instance_ids.append(instance_id)
logging.success(
f'Create integration {integration_name} succeed')
__delete_integrations_instances(c, instance_ids)
return failed_integrations, integrations_counter
def execute_testing(tests_settings: SettingsTester, server_ip: str,
all_tests: set, tests_data_keeper: DataKeeperTester):
"""
Main function used to handle the testing process. Starts by turning off telemetry and disabling
any left over tests. Afterwards it will create a test queue object which then is used to run the
specific test scenario.
:param tests_settings: SettingsTester object which contains the test variables
:param server_ip: IP address of the server. Will be formatted before use.
:param all_tests: All tests currently in the test conf.
:param tests_data_keeper: Object containing all the test results. Used by report tests function.
:return: No object is returned, just updates the tests_data_keep object.
"""
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion or ''
logging.info(
f"Executing tests with the server {server} - and the server ip {server_ip}"
)
slack = tests_settings.slack
circle_ci = tests_settings.circleci
build_number = tests_settings.buildNumber
build_name = tests_settings.buildName
conf, secret_conf = load_conf_files(tests_settings.conf_path,
tests_settings.secret_conf_path)
demisto_api_key = tests_settings.api_key
demisto_user = secret_conf['username']
demisto_pass = secret_conf['userPassword']
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests = extract_filtered_tests()
if not tests or len(tests) == 0:
logging.info('no integrations are configured for test')
return
xsoar_client = demisto_client.configure(base_url=server,
username=demisto_user,
password=demisto_pass,
verify_ssl=False)
# turn off telemetry
turn_off_telemetry(xsoar_client)
failed_playbooks: list = []
succeed_playbooks: list = []
skipped_tests: set = set([])
skipped_integration: set = set([])
playbook_skipped_integration: set = set([])
# Private builds do not use mocking. Here we copy the mocked test list to the unmockable list.
private_tests = get_test_records_of_given_test_names(
tests_settings, all_tests)
try:
# first run the mock tests to avoid mockless side effects in container
logging.info("\nRunning private tests")
executed_in_current_round, private_tests_queue = initialize_queue_and_executed_tests_set(
private_tests)
while not private_tests_queue.empty():
t = private_tests_queue.get()
executed_in_current_round = update_round_set_and_sleep_if_round_completed(
executed_in_current_round, t)
run_private_test_scenario(
tests_settings, t, default_test_timeout, skipped_tests_conf,
nightly_integrations, skipped_integrations_conf,
skipped_integration, filtered_tests, skipped_tests,
secret_params, failed_playbooks, playbook_skipped_integration,
succeed_playbooks, slack, circle_ci, build_number, server,
build_name, server_numeric_version, demisto_user, demisto_pass,
demisto_api_key)
except Exception:
logging.exception('~~ Thread Failed ~~')
raise
finally:
tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks,
skipped_tests, skipped_integration,
unmockable_integrations)
def get_workflow_status(github_token: str,
workflow_id: str) -> Tuple[str, str, str]:
""" Returns a set with the workflow job status, job conclusion and current step that running now in the job
for the given workflow id.
Args:
github_token: Github bearer token.
workflow_id: Github workflow id.
Returns: (Workflow job status, Workflow job conclusion - only if the job completed otherwise its None,
Current step that running now - only if the job is running otherwise its None )
"""
# get the workflow run status
workflow_url = GET_WORKFLOW_URL.format(workflow_id)
res = requests.get(workflow_url,
headers={'Authorization': f'Bearer {github_token}'},
verify=False)
if res.status_code != 200:
logging.critical(
f'Failed to gets private repo workflow, request to {workflow_url} failed with error: {str(res.content)}'
)
sys.exit(1)
# parse response
try:
workflow = json.loads(res.content)
except ValueError:
logging.exception('Enable to parse private repo workflows response')
sys.exit(1)
# get the workflow job from the response to know what step is in progress now
jobs = workflow.get('jobs', [])
if not jobs:
logging.critical(
f'Failed to gets private repo workflow jobs, build url: {WORKFLOW_HTML_URL}/{workflow_id}'
)
sys.exit(1)
curr_job = jobs[0]
job_status = curr_job.get('status')
job_conclusion = curr_job.get('conclusion')
if job_status == 'completed':
return 'completed', job_conclusion, ''
# check for failure steps
failure_steps = [
step for step in jobs[0].get('steps')
if step.get('conclusion') == 'failure'
]
if failure_steps:
return 'completed', 'failure', failure_steps[0].get('name')
# if the job is still in progress - get the current step
curr_step = next(step for step in jobs[0].get('steps')
if step.get('status') == 'in_progress')
return job_status, job_conclusion, curr_step.get('name')
logging.warning(
f'The request to install packs on server {host} has failed due to timeout awaiting response'
f' headers while trying to install pack {e.pack_id}, trying again for one more time'
)
call_install_packs_request(packs_to_install)
except GeneralItemNotFoundError as e:
logging.warning(
f'The request to install all packs on server {host} has failed due to an item not found '
f'error, with the message: {e.error_msg}.\n trying again for one more time'
)
call_install_packs_request(packs_to_install)
except Exception as e:
logging.exception(
f'The request to install packs has failed. Additional info: {str(e)}'
)
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
return SUCCESS_FLAG
def search_pack_and_its_dependencies(client: demisto_client, pack_id: str,
packs_to_install: list,
installation_request_body: list,
lock: Lock):
""" Searches for the pack of the specified file path, as well as its dependencies,
and updates the list of packs to be installed accordingly.
