def get_latest_version_from_bucket(pack_id: str, production_bucket: Bucket) -> str:
""" Retrieves the latest version of pack in the bucket
Args:
pack_id (str): The pack id to retrieve the latest version
production_bucket (Bucket): The GCS production bucket
Returns: The latest version of the pack as it is in the production bucket
"""
pack_bucket_path = os.path.join(GCPConfig.PRODUCTION_STORAGE_BASE_PATH, pack_id)
logging.debug(f'Trying to get latest version for pack {pack_id} from bucket path {pack_bucket_path}')
# Adding the '/' in the end of the prefix to search for the exact pack id
pack_versions_paths = [f.name for f in production_bucket.list_blobs(prefix=f'{pack_bucket_path}/') if
f.name.endswith('.zip')]
pack_versions = []
for path in pack_versions_paths:
versions = PACK_PATH_VERSION_REGEX.findall(path)
if not versions:
continue
pack_versions.append(Version(versions[0]))
logging.debug(f'Found the following zips for {pack_id} pack: {pack_versions}')
if pack_versions:
pack_latest_version = str(max(pack_versions))
return pack_latest_version
else:
logging.error(f'Could not find any versions for pack {pack_id} in bucket path {pack_bucket_path}')
return ''
def install_all_content_packs_for_nightly(client: demisto_client, host: str, service_account: str):
""" Iterates over the packs currently located in the Packs directory. Wrapper for install_packs.
Retrieving the latest version of each pack from the production bucket.
:param client: Demisto-py client to connect to the server.
:param host: FQDN of the server.
:param service_account: The full path to the service account json.
:return: None. Prints the response from the server in the build.
"""
all_packs = []
# Initiate the GCS client and get the production bucket
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(GCPConfig.PRODUCTION_BUCKET)
logging.debug(f"Installing all content packs for nightly flow in server {host}")
# Add deprecated packs to IGNORED_FILES list:
for pack_id in os.listdir(PACKS_FULL_PATH):
if is_pack_hidden(pack_id):
logging.debug(f'Skipping installation of hidden pack "{pack_id}"')
IGNORED_FILES.append(pack_id)
for pack_id in os.listdir(PACKS_FULL_PATH):
if pack_id not in IGNORED_FILES:
pack_version = get_latest_version_from_bucket(pack_id, production_bucket)
if pack_version:
all_packs.append(get_pack_installation_request_data(pack_id, pack_version))
install_packs(client, host, all_packs, is_nightly=True)
def calculate_single_pack_dependencies(pack: str, dependency_graph: object) -> Tuple[dict, list, str]:
"""
Calculates pack dependencies given a pack and a dependencies graph.
First is extract the dependencies subgraph of the given graph only using DFS algorithm with the pack as source.
Then, for all the dependencies of that pack it Replaces the 'mandatory_for_packs' key with a boolean key 'mandatory'
which indicates whether this dependency is mandatory for this pack or not.
Then using that subgraph we get the first-level dependencies and all-levels dependencies.
Args:
pack: The pack for which we need to calculate the dependencies
dependency_graph: The full dependencies graph
Returns:
first_level_dependencies: A dict of the form {'dependency_name': {'mandatory': < >, 'display_name': < >}}
all_level_dependencies: A list with all dependencies names
pack: The pack name
"""
install_logging('Calculate_Packs_Dependencies.log', include_process_name=True, logger=logging)
first_level_dependencies = {}
all_level_dependencies = []
try:
logging.info(f"Calculating {pack} pack dependencies.")
subgraph = PackDependencies.get_dependencies_subgraph_by_dfs(dependency_graph, pack)
for dependency_pack, additional_data in subgraph.nodes(data=True):
logging.debug(f'Iterating dependency {dependency_pack} for pack {pack}')
additional_data['mandatory'] = pack in additional_data['mandatory_for_packs']
del additional_data['mandatory_for_packs']
first_level_dependencies, all_level_dependencies = parse_for_pack_metadata(subgraph, pack)
except Exception:
logging.exception(f"Failed calculating {pack} pack dependencies")
raise
return first_level_dependencies, all_level_dependencies, pack
def call_install_packs_request(packs):
try:
logging.debug(
f'Installing the following packs on server {host}:\n{[pack["id"] for pack in packs]}'
)
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/install',
method='POST',
body={
'packs': packs,
'ignoreWarnings': True
},
accept='application/json',
_request_timeout=request_timeout)
if status_code in range(200, 300) and status_code != 204:
packs_data = [{
'ID': pack.get('id'),
'CurrentVersion': pack.get('currentVersion')
} for pack in ast.literal_eval(response_data)]
logging.success(
f'Packs were successfully installed on server {host}')
logging.debug(
f'The packs that were successfully installed on server {host}:\n{packs_data}'
)
except ApiException as ex:
if 'timeout awaiting response' in ex.body:
raise GCPTimeOutException(ex.body)
if malformed_ids := find_malformed_pack_id(ex.body):
raise MalformedPackException(malformed_ids)
if 'Item not found' in ex.body:
raise GeneralItemNotFoundError(ex.body)
raise ex
def search_pack(client: demisto_client, pack_display_name: str, pack_id: str,
lock: Lock) -> dict:
""" Make a pack search request.
Args:
client (demisto_client): The configured client to use.
pack_display_name (string): The pack display name.
pack_id (string): The pack ID.
lock (Lock): A lock object.
Returns:
(dict): Returns the pack data if found, or empty dict otherwise.
"""
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path=f'/contentpacks/marketplace/{pack_id}',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
logging.debug(
f'Found pack "{pack_display_name}" by its ID "{pack_id}" in bucket!'
)
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
return pack_data
else:
raise Exception(
f'Did not find pack "{pack_display_name}" by its ID "{pack_id}" in bucket.'
)
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception:
logging.exception(
f'Search request for pack "{pack_display_name}" with ID "{pack_id}", failed.'
)
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
return {}
def get_pack_dependencies(client: demisto_client, pack_data: dict, lock: Lock):
""" Get the pack's required dependencies.
Args:
client (demisto_client): The configured client to use.
pack_data (dict): Contains the pack ID and version.
lock (Lock): A lock object.
Returns:
(list) The pack's dependencies.
"""
pack_id = pack_data['id']
logging.debug(f'Getting dependencies for pack {pack_id}')
try:
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/search/dependencies',
method='POST',
body=[pack_data],
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
dependencies_data: list = []
dependants_ids = [pack_id]
reseponse_data = ast.literal_eval(response_data).get(
'dependencies', [])
create_dependencies_data_structure(reseponse_data, dependants_ids,
dependencies_data,
dependants_ids)
dependencies_str = ', '.join(
[dep['id'] for dep in dependencies_data])
if dependencies_data:
logging.debug(
f'Found the following dependencies for pack {pack_id}: {dependencies_str}'
)
return dependencies_data
if status_code == 400:
logging.error(f'Unable to find dependencies for {pack_id}.')
return []
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
raise Exception(
f'Failed to get pack {pack_id} dependencies - with status code {status_code}\n{msg}\n'
)
except Exception:
logging.exception(
f'The request to get pack {pack_id} dependencies has failed.')
lock.acquire()
global SUCCESS_FLAG
SUCCESS_FLAG = False
lock.release()
def install_packs(client: demisto_client,
host: str,
packs_to_install: list,
request_timeout: int = 999999,
is_nightly: bool = False):
""" Make a packs installation request.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
is_nightly (bool): Is the build nightly or not.
"""
if is_nightly:
install_nightly_packs(client, host, packs_to_install)
return
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
logging.info(f'Installing packs on server {host}')
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs on server {host}:\n{packs_to_install_str}')
# make the pack installation request
try:
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for
pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception as e:
logging.exception(f'The request to install packs has failed. Additional info: {str(e)}')
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
return SUCCESS_FLAG
def uninstall_all_packs(client: demisto_client, hostname):
""" Lists all installed packs and uninstalling them.
Args:
client (demisto_client): The client to connect to.
hostname (str): xsiam hostname
Returns (list, bool):
A flag that indicates if the operation succeeded or not.
"""
logging.info(
f'Starting to search and uninstall packs in server: {hostname}')
packs_to_uninstall: list = get_all_installed_packs(client)
if packs_to_uninstall:
return uninstall_packs(client, packs_to_uninstall)
logging.debug('Skipping packs uninstallation - nothing to uninstall')
return True
def reset_base_pack_version(client: demisto_client):
"""
Resets base pack version to prod version.
Args:
client (demisto_client): The client to connect to.
"""
host = client.api_client.configuration.host.replace(
'https://api-', 'https://') # disable-secrets-detection
try:
# make the search request
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/Base',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
result_object = ast.literal_eval(response_data)
if result_object and result_object.get('currentVersion'):
logging.debug('Found Base pack in bucket!')
pack_data = {
'id': result_object.get('id'),
'version': result_object.get('currentVersion')
}
# install latest version of Base pack
logging.info(
f'updating base pack to version {result_object.get("currentVersion")}'
)
return install_packs(client, host, [pack_data], False)
else:
raise Exception('Did not find Base pack')
else:
result_object = ast.literal_eval(response_data)
msg = result_object.get('message', '')
err_msg = f'Search request for base pack, failed with status code ' \
f'{status_code}\n{msg}'
raise Exception(err_msg)
except Exception:
logging.exception('Search request Base pack has failed.')
return False
def add_pack_metadata_results(results: Tuple) -> None:
"""
This is a callback that should be called once the result of the future is ready.
The results include: first_level_dependencies, all_level_dependencies, pack_name
Using these results we write the dependencies
"""
try:
first_level_dependencies, all_level_dependencies, pack_name = results
logging.debug(f'Got dependencies for pack {pack_name}\n: {pformat(all_level_dependencies)}')
pack_dependencies_result[pack_name] = {
"dependencies": first_level_dependencies,
"displayedImages": list(first_level_dependencies.keys()),
"allLevelDependencies": all_level_dependencies,
"path": os.path.join(PACKS_FOLDER, pack_name),
"fullPath": os.path.abspath(os.path.join(PACKS_FOLDER, pack_name))
}
except Exception:
logging.exception('Failed to collect pack dependencies results')
raise
def get_all_installed_packs(client: demisto_client):
"""
Args:
client (demisto_client): The client to connect to.
Returns:
list of installed python
"""
try:
logging.info("Attempting to fetch all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/metadata/installed',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
installed_packs = ast.literal_eval(response_data)
installed_packs_ids = [pack.get('id') for pack in installed_packs]
logging.success('Successfully fetched all installed packs.')
installed_packs_ids_str = ', '.join(installed_packs_ids)
logging.debug(
f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}'
)
if 'Base' in installed_packs_ids:
installed_packs_ids.remove('Base')
return installed_packs_ids
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(
f'Failed to fetch installed packs - with status code {status_code}\n{message}'
)
except Exception as e:
logging.exception(
f'The request to fetch installed packs has failed. Additional info: {str(e)}'
)
return None
def check_index_data(index_data: dict) -> bool:
"""Check index.json file inside the index.zip archive in the cloud.
Validate by running verify_pack on each pack.
Args:
index_data: Dictionary of the index.json contents.
Returns: True if all packs are valid, False otherwise.
"""
logging.info("Found index data in index file. Checking...")
logging.debug(f"Index data is:\n {pformat(index_data)}")
packs_list_exists = log_message_if_statement(
statement=(len(index_data.get("packs", [])) != 0),
error_message="Found 0 packs in index file."
"\nAborting the rest of the check.")
# If all packs are gone, return False
if not packs_list_exists:
return False
mandatory_pack_ids = load_json(MANDATORY_PREMIUM_PACKS_PATH).get(
"packs", [])
packs_are_valid = True
for pack in index_data["packs"]:
pack_is_good = verify_pack(pack)
if not pack_is_good:
packs_are_valid = False
if pack["id"] in mandatory_pack_ids:
mandatory_pack_ids.remove(pack["id"])
all_mandatory_packs_are_found = log_message_if_statement(
statement=(mandatory_pack_ids == []),
error_message=f"index json is missing some mandatory"
f" pack ids: {pformat(mandatory_pack_ids)}",
success_message="All premium mandatory pack ids were"
" found in the index.json file.")
return all([packs_are_valid, all_mandatory_packs_are_found])
def install_all_content_packs_from_build_bucket(client: demisto_client,
host: str, server_version: str,
bucket_packs_root_path: str,
service_account: str,
extract_destination_path: str):
""" Iterates over the packs currently located in the Build bucket. Wrapper for install_packs.
Retrieving the metadata of the latest version of each pack from the index.zip of the build bucket.
:param client: Demisto-py client to connect to the server.
:param host: FQDN of the server.
:param server_version: The version of the server the packs are installed on.
:param bucket_packs_root_path: The prefix to the root of packs in the bucket
:param service_account: Google Service Account
:param extract_destination_path: the full path of extract folder for the index.
:return: None. Prints the response from the server in the build.
"""
all_packs = []
logging.debug(
f"Installing all content packs in server {host} from packs path {bucket_packs_root_path}"
)
storage_client = init_storage_client(service_account)
build_bucket = storage_client.bucket(GCPConfig.CI_BUILD_BUCKET)
index_folder_path, _, _ = download_and_extract_index(
build_bucket, extract_destination_path, bucket_packs_root_path)
for pack_id in os.listdir(index_folder_path):
if os.path.isdir(os.path.join(index_folder_path, pack_id)):
metadata_path = os.path.join(index_folder_path, pack_id,
Pack.METADATA)
pack_metadata = load_json(metadata_path)
if 'partnerId' in pack_metadata: # not installing private packs
continue
pack_version = pack_metadata.get(
Metadata.CURRENT_VERSION, Metadata.SERVER_DEFAULT_MIN_VERSION)
server_min_version = pack_metadata.get(
Metadata.SERVER_MIN_VERSION,
Metadata.SERVER_DEFAULT_MIN_VERSION)
hidden = pack_metadata.get(Metadata.HIDDEN, False)
# Check if the server version is greater than the minimum server version required for this pack or if the
# pack is hidden (deprecated):
if ('Master' in server_version or Version(server_version) >= Version(server_min_version)) and \
not hidden:
logging.debug(f"Appending pack id {pack_id}")
all_packs.append(
get_pack_installation_request_data(pack_id, pack_version))
else:
reason = 'Is hidden' if hidden else f'min server version is {server_min_version}'
logging.debug(
f'Pack: {pack_id} with version: {pack_version} will not be installed on {host}. '
f'Pack {reason}.')
return install_packs(client, host, all_packs)
def install_nightly_packs(client: demisto_client,
host: str,
packs_to_install: List,
request_timeout: int = 999999):
"""
Install content packs on nightly build.
We will catch the exception if pack fails to install and send the request to install packs again without the
corrupted pack.
Args:
client(demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
Returns:
None: No data returned.
"""
logging.info(f'Installing packs on server {host}')
# make the pack installation request
all_packs_install_successfully = False
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
while not all_packs_install_successfully:
try:
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs in server {host}:\n{packs_to_install_str}')
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs on server {host}- with status code {status_code}\n{message}\n')
break
except Exception as e:
all_packs_install_successfully = False
malformed_pack_id = find_malformed_pack_id(str(e))
if not malformed_pack_id:
logging.exception('The request to install packs has failed')
raise
pack_ids_to_install = {pack['id'] for pack in packs_to_install}
malformed_pack_id = malformed_pack_id[0]
if malformed_pack_id not in pack_ids_to_install:
logging.exception(
f'The pack {malformed_pack_id} has failed to install even though it was not in the installation list')
raise
logging.warning(f'The request to install packs on server {host} has failed, retrying without {malformed_pack_id}')
# Remove the malformed pack from the pack to install list.
packs_to_install = [pack for pack in packs_to_install if pack['id'] not in malformed_pack_id]
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
def main():
install_logging('Copy_and_Upload_Packs.log', logger=logging)
options = options_handler()
packs_artifacts_path = options.artifacts_path
extract_destination_path = options.extract_path
production_bucket_name = options.production_bucket_name
build_bucket_name = options.build_bucket_name
service_account = options.service_account
build_number = options.ci_build_number
circle_branch = options.circle_branch
production_base_path = options.production_base_path
target_packs = options.pack_names
marketplace = options.marketplace
# Google cloud storage client initialized
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(production_bucket_name)
build_bucket = storage_client.bucket(build_bucket_name)
# Initialize build and prod base paths
build_bucket_path = os.path.join(GCPConfig.BUILD_PATH_PREFIX,
circle_branch, build_number, marketplace)
build_bucket_base_path = os.path.join(build_bucket_path,
GCPConfig.CONTENT_PACKS_PATH)
# Relevant when triggering test upload flow
if production_bucket_name:
GCPConfig.PRODUCTION_BUCKET = production_bucket_name
# Download and extract build index from build and prod buckets
build_index_folder_path, build_index_blob, build_index_generation = \
download_and_extract_index(build_bucket, extract_destination_path, build_bucket_base_path)
# Get the successful and failed packs file from Prepare Content step in Create Instances job if there are
packs_results_file_path = os.path.join(
os.path.dirname(packs_artifacts_path),
BucketUploadFlow.PACKS_RESULTS_FILE)
pc_successful_packs_dict, pc_failed_packs_dict, pc_successful_private_packs_dict, \
pc_uploaded_images = get_upload_data(packs_results_file_path, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
logging.debug(
f"Successful packs from Prepare Content: {pc_successful_packs_dict}")
logging.debug(f"Failed packs from Prepare Content: {pc_failed_packs_dict}")
logging.debug(
f"Successful private packs from Prepare Content: {pc_successful_private_packs_dict}"
)
logging.debug(f"Images from Prepare Content: {pc_uploaded_images}")
# Check if needs to upload or not
check_if_need_to_upload(pc_successful_packs_dict, pc_failed_packs_dict,
pc_successful_private_packs_dict,
pc_uploaded_images)
# Detect packs to upload
pack_names = get_pack_names(target_packs)
extract_packs_artifacts(packs_artifacts_path, extract_destination_path)
packs_list = [
Pack(pack_name, os.path.join(extract_destination_path, pack_name),
marketplace) for pack_name in pack_names
if os.path.exists(os.path.join(extract_destination_path, pack_name))
]
# Starting iteration over packs
for pack in packs_list:
# Indicates whether a pack has failed to upload on Prepare Content step
task_status, pack_status = pack.is_failed_to_upload(
pc_failed_packs_dict)
if task_status:
pack.status = pack_status
pack.cleanup()
continue
task_status = pack.load_user_metadata(marketplace)
if not task_status:
pack.status = PackStatus.FAILED_LOADING_USER_METADATA.name
pack.cleanup()
continue
task_status = pack.copy_integration_images(production_bucket,
build_bucket,
pc_uploaded_images,
production_base_path,
build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
continue
task_status = pack.copy_author_image(production_bucket, build_bucket,
pc_uploaded_images,
production_base_path,
build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
continue
task_status, skipped_pack_uploading = pack.copy_and_upload_to_storage(
production_bucket, build_bucket, pc_successful_packs_dict,
production_base_path, build_bucket_base_path)
if skipped_pack_uploading:
pack.status = PackStatus.PACK_ALREADY_EXISTS.name
pack.cleanup()
continue
if not task_status:
pack.status = PackStatus.FAILED_UPLOADING_PACK.name
pack.cleanup()
continue
pack.status = PackStatus.SUCCESS.name
# upload core packs json to bucket
upload_core_packs_config(production_bucket, build_number,
extract_destination_path, build_bucket,
production_base_path, build_bucket_base_path)
# finished iteration over content packs
copy_index(build_index_folder_path, build_index_blob,
build_index_generation, production_bucket, build_bucket,
production_base_path, build_bucket_base_path)
# upload id_set.json to bucket
copy_id_set(production_bucket, build_bucket, production_base_path,
build_bucket_base_path)
# get the lists of packs divided by their status
successful_packs, skipped_packs, failed_packs = get_packs_summary(
packs_list)
# Store successful and failed packs list in CircleCI artifacts
store_successful_and_failed_packs_in_ci_artifacts(
packs_results_file_path,
BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE, successful_packs,
failed_packs, list(pc_successful_private_packs_dict))
# verify that the successful from Prepare content and are the ones that were copied
verify_copy(successful_packs, pc_successful_packs_dict)
# summary of packs status
print_packs_summary(successful_packs, skipped_packs, failed_packs)
