def dummy_pack_metadata():
""" Fixture for dummy pack_metadata.json file that is part of pack folder in content repo.
"""
dummy_pack_metadata_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data",
"user_pack_metadata.json")
pack_metadata = load_json(dummy_pack_metadata_path)
return pack_metadata
def upload_core_packs_config(production_bucket: Bucket, build_number: str,
extract_destination_path: str,
build_bucket: Bucket):
"""Uploads corepacks.json file configuration to bucket. Corepacks file includes core packs for server installation.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is uploaded.
build_number (str): CircleCI build number.
extract_destination_path (str): Full path of folder to extract the corepacks file
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is downloaded from.
"""
# download the corepacks.json stored in the build bucket to temp dir
build_corepacks_file_path = os.path.join(GCPConfig.BUILD_BASE_PATH,
GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob = build_bucket.blob(build_corepacks_file_path)
if not build_corepacks_blob.exists():
logging.critical(
f"{GCPConfig.CORE_PACK_FILE_NAME} is missing in {build_bucket.name} bucket, exiting..."
)
sys.exit(1)
temp_corepacks_file_path = os.path.join(extract_destination_path,
GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob.download_to_filename(temp_corepacks_file_path)
corepacks_file = load_json(temp_corepacks_file_path)
# change the storage paths to the prod bucket
corepacks_list = corepacks_file.get('corePacks', [])
try:
corepacks_list = [
os.path.join(GCPConfig.GCS_PUBLIC_URL, production_bucket.name,
GCPConfig.STORAGE_BASE_PATH,
LATEST_ZIP_REGEX.findall(corepack_path)[0])
for corepack_path in corepacks_list
]
except IndexError:
corepacks_list_str = '\n'.join(corepacks_list)
logging.exception(
f"GCS paths in build bucket corepacks.json file are not of format: "
f"{GCPConfig.GCS_PUBLIC_URL}/<BUCKET_NAME>/.../content/packs/...\n"
f"List of build bucket corepacks paths:\n{corepacks_list_str}")
sys.exit(1)
# construct core pack data with public gcs urls
core_packs_data = {
'corePacks': corepacks_list,
'buildNumber': build_number
}
# upload core pack json file to gcs
prod_corepacks_file_path = os.path.join(GCPConfig.STORAGE_BASE_PATH,
GCPConfig.CORE_PACK_FILE_NAME)
prod_corepacks_blob = production_bucket.blob(prod_corepacks_file_path)
prod_corepacks_blob.upload_from_string(
json.dumps(core_packs_data, indent=4))
logging.success(
f"Finished uploading {GCPConfig.CORE_PACK_FILE_NAME} to storage.")
def get_updated_private_packs(private_packs, index_folder_path):
""" Checks for updated private packs by compering contentCommitHash between public index json and private pack
metadata files.
Args:
private_packs (list): List of dicts containing pack metadata information.
index_folder_path (str): The public index folder path.
Returns:
updated_private_packs (list) : a list of all private packs id's that were updated.
"""
updated_private_packs = []
public_index_file_path = os.path.join(index_folder_path,
f"{GCPConfig.INDEX_NAME}.json")
public_index_json = load_json(public_index_file_path)
private_packs_from_public_index = public_index_json.get("packs", {})
for pack in private_packs:
private_pack_id = pack.get('id')
private_commit_hash_from_metadata = pack.get('contentCommitHash', "")
private_commit_hash_from_content_repo = ""
for public_pack in private_packs_from_public_index:
if public_pack.get('id') == private_pack_id:
private_commit_hash_from_content_repo = public_pack.get(
'contentCommitHash', "")
private_pack_was_updated = private_commit_hash_from_metadata != private_commit_hash_from_content_repo
if private_pack_was_updated:
updated_private_packs.append(private_pack_id)
return updated_private_packs
def get_index_json_data(service_account: str, production_bucket_name: str, extract_path: str, storage_base_path: str) \
-> (dict, str):
"""Retrieve the index.json file from production bucket.
Args:
service_account: Path to gcloud service account
production_bucket_name: Production bucket name
extract_path: Full path of folder to extract the index.zip to
storage_base_path: The base path in the bucket
Returns:
(Dict: content of the index.json, Str: path to index.json)
"""
logging.info('Downloading and extracting index.zip from the cloud')
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(production_bucket_name)
index_folder_path, _, _ = download_and_extract_index(
production_bucket, extract_path, storage_base_path)
logging.info("Retrieving the index file")
index_file_path = os.path.join(index_folder_path,
f"{GCPConfig.INDEX_NAME}.json")
index_data = load_json(index_file_path)
return index_data, index_file_path
def install_all_content_packs_from_build_bucket(client: demisto_client,
host: str, server_version: str,
bucket_packs_root_path: str,
service_account: str,
extract_destination_path: str):
""" Iterates over the packs currently located in the Build bucket. Wrapper for install_packs.
Retrieving the metadata of the latest version of each pack from the index.zip of the build bucket.
:param client: Demisto-py client to connect to the server.
:param host: FQDN of the server.
:param server_version: The version of the server the packs are installed on.
:param bucket_packs_root_path: The prefix to the root of packs in the bucket
:param service_account: Google Service Account
:param extract_destination_path: the full path of extract folder for the index.
:return: None. Prints the response from the server in the build.
"""
all_packs = []
logging.debug(
f"Installing all content packs in server {host} from packs path {bucket_packs_root_path}"
)
storage_client = init_storage_client(service_account)
build_bucket = storage_client.bucket(GCPConfig.CI_BUILD_BUCKET)
index_folder_path, _, _ = download_and_extract_index(
build_bucket, extract_destination_path, bucket_packs_root_path)
for pack_id in os.listdir(index_folder_path):
if os.path.isdir(os.path.join(index_folder_path, pack_id)):
metadata_path = os.path.join(index_folder_path, pack_id,
Pack.METADATA)
pack_metadata = load_json(metadata_path)
if 'partnerId' in pack_metadata: # not installing private packs
continue
pack_version = pack_metadata.get(
Metadata.CURRENT_VERSION, Metadata.SERVER_DEFAULT_MIN_VERSION)
server_min_version = pack_metadata.get(
Metadata.SERVER_MIN_VERSION,
Metadata.SERVER_DEFAULT_MIN_VERSION)
hidden = pack_metadata.get(Metadata.HIDDEN, False)
# Check if the server version is greater than the minimum server version required for this pack or if the
# pack is hidden (deprecated):
if ('Master' in server_version or LooseVersion(server_version) >= LooseVersion(server_min_version)) and \
not hidden:
logging.debug(f"Appending pack id {pack_id}")
all_packs.append(
get_pack_installation_request_data(pack_id, pack_version))
else:
reason = 'Is hidden' if hidden else f'min server version is {server_min_version}'
logging.debug(
f'Pack: {pack_id} with version: {pack_version} will not be installed on {host}. '
f'Pack {reason}.')
return install_packs(client, host, all_packs)
def extract_credentials_from_secret(secret_path: str) -> (str, str):
"""Extract Credentials from secret file.
Args:
secret_path: The path to the secret file.
Returns: (username, password) found in the secret file.
"""
logging.info("Retrieving the credentials for Cortex XSOAR server")
secret_conf_file = load_json(file_path=secret_path)
username: str = secret_conf_file.get("username")
password: str = secret_conf_file.get("userPassword")
return username, password
def get_successful_and_failed_packs(
packs_results_file_path: str) -> Tuple[dict, dict]:
""" Loads the packs_results.json file to get the successful and failed packs dicts
Args:
packs_results_file_path: The path to the file
Returns:
dict: The successful packs dict
dict: The failed packs dict
"""
if os.path.exists(packs_results_file_path):
packs_results_file = load_json(packs_results_file_path)
successful_packs_dict = packs_results_file.get('successful_packs', {})
failed_packs_dict = packs_results_file.get('failed_packs', {})
return successful_packs_dict, failed_packs_dict
return {}, {}
def check_index_data(index_data: dict) -> bool:
"""Check index.json file inside the index.zip archive in the cloud.
Validate by running verify_pack on each pack.
Args:
index_data: Dictionary of the index.json contents.
Returns: True if all packs are valid, False otherwise.
"""
logging.info("Found index data in index file. Checking...")
logging.debug(f"Index data is:\n {pformat(index_data)}")
packs_list_exists = log_message_if_statement(
statement=(len(index_data.get("packs", [])) != 0),
error_message="Found 0 packs in index file."
"\nAborting the rest of the check.")
# If all packs are gone, return False
if not packs_list_exists:
return False
mandatory_pack_ids = load_json(MANDATORY_PREMIUM_PACKS_PATH).get(
"packs", [])
packs_are_valid = True
for pack in index_data["packs"]:
pack_is_good = verify_pack(pack)
if not pack_is_good:
packs_are_valid = False
if pack["id"] in mandatory_pack_ids:
mandatory_pack_ids.remove(pack["id"])
all_mandatory_packs_are_found = log_message_if_statement(
statement=(mandatory_pack_ids == []),
error_message=f"index json is missing some mandatory"
f" pack ids: {pformat(mandatory_pack_ids)}",
success_message="All premium mandatory pack ids were"
" found in the index.json file.")
return all([packs_are_valid, all_mandatory_packs_are_found])
def handle_private_content(public_index_folder_path, private_bucket_name, extract_destination_path, storage_client,
public_pack_names) -> Tuple[bool, list, list]:
"""
1. Add private packs to public index.json.
2. Checks if there are private packs that were added/deleted/updated.
Args:
public_index_folder_path: extracted public index folder full path.
private_bucket_name: Private storage bucket name
extract_destination_path: full path to extract directory.
storage_client : initialized google cloud storage client.
public_pack_names : unique collection of public packs names to upload.
Returns:
is_private_content_updated (bool): True if there is at least one private pack that was updated/released.
False otherwise (i.e there are no private packs that have been updated/released).
private_packs (list) : priced packs from private bucket.
updated_private_packs_ids (list): all private packs id's that were updated.
"""
if private_bucket_name:
private_storage_bucket = storage_client.bucket(private_bucket_name)
private_index_path, _, _ = download_and_extract_index(
private_storage_bucket, os.path.join(extract_destination_path, "private")
)
public_index_json_file_path = os.path.join(public_index_folder_path, f"{GCPConfig.INDEX_NAME}.json")
public_index_json = load_json(public_index_json_file_path)
if public_index_json:
are_private_packs_updated = is_private_packs_updated(public_index_json, private_index_path)
private_packs, updated_private_packs_ids = add_private_content_to_index(
private_index_path, extract_destination_path, public_index_folder_path, public_pack_names
)
return are_private_packs_updated, private_packs, updated_private_packs_ids
else:
logging.error(f"Public {GCPConfig.INDEX_NAME}.json was found empty.")
sys.exit(1)
else:
return False, [], []
def is_private_packs_updated(public_index_json, private_index_path):
""" Checks whether there were changes in private packs from the last upload.
The check compares the `content commit hash` field in the public index with the value stored in the private index.
If there is at least one private pack that has been updated/released, the upload should be performed and not
skipped.
Args:
public_index_json (dict) : The public index.json file.
private_index_path (str): Path to where the private index.zip is located.
Returns:
is_private_packs_updated (bool): True if there is at least one private pack that was updated/released,
False otherwise (i.e there are no private packs that have been updated/released).
"""
logging.debug("Checking if there are updated private packs")
private_index_file_path = os.path.join(private_index_path, f"{GCPConfig.INDEX_NAME}.json")
private_index_json = load_json(private_index_file_path)
private_packs_from_private_index = private_index_json.get("packs")
private_packs_from_public_index = public_index_json.get("packs")
if len(private_packs_from_private_index) != len(private_packs_from_public_index):
# private pack was added or deleted
logging.debug("There is at least one private pack that was added/deleted, upload should not be skipped.")
return True
id_to_commit_hash_from_public_index = {private_pack.get("id"): private_pack.get("contentCommitHash", "") for
private_pack in private_packs_from_public_index}
for private_pack in private_packs_from_private_index:
pack_id = private_pack.get("id")
content_commit_hash = private_pack.get("contentCommitHash", "")
if id_to_commit_hash_from_public_index.get(pack_id) != content_commit_hash:
logging.debug("There is at least one private pack that was updated, upload should not be skipped.")
return True
logging.debug("No private packs were changed")
return False
def main():
install_logging('upload_packs_private.log')
upload_config = option_handler()
packs_artifacts_path = upload_config.artifacts_path
extract_destination_path = upload_config.extract_path
storage_bucket_name = upload_config.bucket_name
private_bucket_name = upload_config.private_bucket_name
service_account = upload_config.service_account
target_packs = upload_config.pack_names
build_number = upload_config.ci_build_number
id_set_path = upload_config.id_set_path
packs_dependencies_mapping = load_json(
upload_config.pack_dependencies
) if upload_config.pack_dependencies else {}
storage_base_path = upload_config.storage_base_path
is_private_build = upload_config.encryption_key and upload_config.encryption_key != ''
landing_page_sections = load_json(LANDING_PAGE_SECTIONS_PATH)
logging.info(f"Packs artifact path is: {packs_artifacts_path}")
prepare_test_directories(packs_artifacts_path)
# google cloud storage client initialized
storage_client = init_storage_client(service_account)
storage_bucket = storage_client.bucket(storage_bucket_name)
private_storage_bucket = storage_client.bucket(private_bucket_name)
default_storage_bucket = private_storage_bucket if is_private_build else storage_bucket
# download and extract index from public bucket
index_folder_path, index_blob, index_generation = download_and_extract_index(
storage_bucket, extract_destination_path)
# content repo client initialized
if not is_private_build:
content_repo = get_content_git_client(CONTENT_ROOT_PATH)
current_commit_hash, remote_previous_commit_hash = get_recent_commits_data(
content_repo,
index_folder_path,
is_bucket_upload_flow=False,
is_private_build=True)
else:
current_commit_hash, remote_previous_commit_hash = "", ""
content_repo = None
if storage_base_path:
GCPConfig.STORAGE_BASE_PATH = storage_base_path
# detect packs to upload
pack_names = get_packs_names(target_packs)
extract_packs_artifacts(packs_artifacts_path, extract_destination_path)
packs_list = [
Pack(pack_name, os.path.join(extract_destination_path, pack_name))
for pack_name in pack_names
if os.path.exists(os.path.join(extract_destination_path, pack_name))
]
if not is_private_build:
check_if_index_is_updated(index_folder_path, content_repo,
current_commit_hash,
remote_previous_commit_hash, storage_bucket)
if private_bucket_name: # Add private packs to the index
private_packs, private_index_path, private_index_blob = update_index_with_priced_packs(
private_storage_bucket, extract_destination_path,
index_folder_path, pack_names, is_private_build)
else: # skipping private packs
logging.info("Skipping index update of priced packs")
private_packs = []
# google cloud bigquery client initialized
packs_statistic_df = None
if not is_private_build:
bq_client = init_bigquery_client(service_account)
packs_statistic_df = get_packs_statistics_dataframe(bq_client)
# clean index and gcs from non existing or invalid packs
clean_non_existing_packs(index_folder_path, private_packs,
default_storage_bucket)
# starting iteration over packs
for pack in packs_list:
create_and_upload_marketplace_pack(
upload_config,
pack,
storage_bucket,
index_folder_path,
packs_dependencies_mapping,
private_bucket_name,
private_storage_bucket=private_storage_bucket,
content_repo=content_repo,
current_commit_hash=current_commit_hash,
remote_previous_commit_hash=remote_previous_commit_hash,
packs_statistic_df=packs_statistic_df)
# upload core packs json to bucket
if should_upload_core_packs(storage_bucket_name):
upload_core_packs_config(default_storage_bucket, build_number,
index_folder_path)
# finished iteration over content packs
if is_private_build:
delete_public_packs_from_index(index_folder_path)
upload_index_to_storage(index_folder_path,
extract_destination_path,
private_index_blob,
build_number,
private_packs,
current_commit_hash,
index_generation,
is_private_build,
landing_page_sections=landing_page_sections)
else:
upload_index_to_storage(index_folder_path,
extract_destination_path,
index_blob,
build_number,
private_packs,
current_commit_hash,
index_generation,
landing_page_sections=landing_page_sections)
# upload id_set.json to bucket
upload_id_set(default_storage_bucket, id_set_path)
# get the lists of packs divided by their status
successful_packs, skipped_packs, failed_packs = get_packs_summary(
packs_list)
# summary of packs status
print_packs_summary(successful_packs, skipped_packs, failed_packs)
def create_and_upload_marketplace_pack(upload_config: Any, pack: Any, storage_bucket: Any, index_folder_path: str,
packs_dependencies_mapping: dict, private_bucket_name: str,
private_storage_bucket: bool = None,
content_repo: bool = None, current_commit_hash: str = '',
remote_previous_commit_hash: str = '', packs_statistic_df: Any = None) \
-> Any:
"""
The main logic flow for the create and upload process. Acts as a decision tree while consistently
checking the status of the progress being made.
:param upload_config: Configuration for the script as handled by the Option Handler.
:param pack: Pack object.
:param storage_bucket: Bucket the changes are being uploaded to.
:param index_folder_path: Path to the index folder.
:param packs_dependencies_mapping: Used by format_metadata to add dependencies to the metadata file.
:param private_storage_bucket: Bucket where the private packs are uploaded.
:param content_repo: The main content repository. demisto/content
:param current_commit_hash: Current commit hash for the run. Used in the pack metadata file.
:param remote_previous_commit_hash: Previous commit hash. Used for comparison.
:param packs_statistic_df: Dataframe object containing current pack analytics.
:return: Updated pack.status value.
"""
build_number = upload_config.ci_build_number
remove_test_playbooks = upload_config.remove_test_playbooks
signature_key = upload_config.key_string
extract_destination_path = upload_config.extract_path
override_all_packs = upload_config.override_all_packs
enc_key = upload_config.encryption_key
packs_artifacts_dir = upload_config.artifacts_path
private_artifacts_dir = upload_config.private_artifacts
is_infra_run = upload_config.is_infra_run
secondary_enc_key = upload_config.secondary_encryption_key
landing_page_sections = load_json(LANDING_PAGE_SECTIONS_PATH)
pack_was_modified = not is_infra_run
task_status, user_metadata = pack.load_user_metadata()
if not task_status:
pack.status = PackStatus.FAILED_LOADING_USER_METADATA.name
pack.cleanup()
return
task_status, pack_content_items = pack.collect_content_items()
if not task_status:
pack.status = PackStatus.FAILED_COLLECT_ITEMS.name
pack.cleanup()
return
task_status, integration_images = pack.upload_integration_images(
storage_bucket)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
return
task_status, author_image = pack.upload_author_image(storage_bucket)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
return
task_status = pack.format_metadata(
user_metadata=user_metadata,
pack_content_items=pack_content_items,
integration_images=integration_images,
author_image=author_image,
index_folder_path=index_folder_path,
packs_dependencies_mapping=packs_dependencies_mapping,
build_number=build_number,
commit_hash=current_commit_hash,
packs_statistic_df=packs_statistic_df,
pack_was_modified=pack_was_modified,
landing_page_sections=landing_page_sections)
if not task_status:
pack.status = PackStatus.FAILED_METADATA_PARSING.name
pack.cleanup()
return
task_status, not_updated_build = pack.prepare_release_notes(
index_folder_path, build_number)
if not task_status:
pack.status = PackStatus.FAILED_RELEASE_NOTES.name
pack.cleanup()
return
if not_updated_build:
pack.status = PackStatus.PACK_IS_NOT_UPDATED_IN_RUNNING_BUILD.name
pack.cleanup()
return
task_status = pack.remove_unwanted_files(remove_test_playbooks)
if not task_status:
pack.status = PackStatus.FAILED_REMOVING_PACK_SKIPPED_FOLDERS
pack.cleanup()
return
task_status = pack.sign_pack(signature_key)
if not task_status:
pack.status = PackStatus.FAILED_SIGNING_PACKS.name
pack.cleanup()
return
task_status, zip_pack_path = pack.zip_pack(extract_destination_path,
pack._pack_name, enc_key,
private_artifacts_dir,
secondary_enc_key)
if not task_status:
pack.status = PackStatus.FAILED_ZIPPING_PACK_ARTIFACTS.name
pack.cleanup()
return
task_status = pack.is_pack_encrypted(zip_pack_path, enc_key)
if not task_status:
pack.status = PackStatus.FAILED_DECRYPT_PACK.name
pack.cleanup()
return
bucket_for_uploading = private_storage_bucket if private_storage_bucket else storage_bucket
(task_status, skipped_pack_uploading, full_pack_path) = \
pack.upload_to_storage(zip_pack_path, pack.latest_version,
bucket_for_uploading, override_all_packs
or pack_was_modified, pack_artifacts_path=packs_artifacts_dir,
private_content=True)
if full_pack_path is not None:
bucket_path = f'https://console.cloud.google.com/storage/browser/{private_bucket_name}/'
bucket_url = bucket_path + full_pack_path
else:
bucket_url = 'Pack was not uploaded.'
pack.bucket_url = bucket_url
if not task_status:
pack.status = PackStatus.FAILED_UPLOADING_PACK.name
pack.cleanup()
return
task_status, exists_in_index = pack.check_if_exists_in_index(
index_folder_path)
if not task_status:
pack.status = PackStatus.FAILED_SEARCHING_PACK_IN_INDEX.name
pack.cleanup()
return
task_status = pack.prepare_for_index_upload()
if not task_status:
pack.status = PackStatus.FAILED_PREPARING_INDEX_FOLDER.name
pack.cleanup()
return
task_status = update_index_folder(index_folder_path=index_folder_path,
pack_name=pack.name,
pack_path=pack.path,
pack_version=pack.latest_version,
hidden_pack=pack.hidden)
if not task_status:
pack.status = PackStatus.FAILED_UPDATING_INDEX_FOLDER.name
pack.cleanup()
return
# in case that pack already exist at cloud storage path and in index, don't show that the pack was changed
if skipped_pack_uploading and exists_in_index:
pack.status = PackStatus.PACK_ALREADY_EXISTS.name
pack.cleanup()
return
pack.status = PackStatus.SUCCESS.name
def get_landing_page_sections() -> Dict:
""" Returns the landing page sections file content """
return mp_services.load_json(LANDING_PAGE_SECTIONS_PATH)
def main():
install_logging('Prepare_Content_Packs_For_Testing.log')
option = option_handler()
packs_artifacts_path = option.artifacts_path
extract_destination_path = option.extract_path
storage_bucket_name = option.bucket_name
service_account = option.service_account
target_packs = option.pack_names if option.pack_names else ""
build_number = option.ci_build_number if option.ci_build_number else str(
uuid.uuid4())
override_all_packs = option.override_all_packs
signature_key = option.key_string
id_set_path = option.id_set_path
packs_dependencies_mapping = load_json(
option.pack_dependencies) if option.pack_dependencies else {}
storage_base_path = option.storage_base_path
remove_test_playbooks = option.remove_test_playbooks
is_bucket_upload_flow = option.bucket_upload
private_bucket_name = option.private_bucket_name
circle_branch = option.circle_branch
force_upload = option.force_upload
# google cloud storage client initialized
storage_client = init_storage_client(service_account)
storage_bucket = storage_client.bucket(storage_bucket_name)
if storage_base_path:
GCPConfig.STORAGE_BASE_PATH = storage_base_path
# download and extract index from public bucket
index_folder_path, index_blob, index_generation = download_and_extract_index(
storage_bucket, extract_destination_path)
# content repo client initialized
content_repo = get_content_git_client(CONTENT_ROOT_PATH)
current_commit_hash, previous_commit_hash = get_recent_commits_data(
content_repo, index_folder_path, is_bucket_upload_flow, circle_branch)
# detect packs to upload
pack_names = get_packs_names(target_packs, previous_commit_hash)
extract_packs_artifacts(packs_artifacts_path, extract_destination_path)
packs_list = [
Pack(pack_name, os.path.join(extract_destination_path, pack_name))
for pack_name in pack_names
if os.path.exists(os.path.join(extract_destination_path, pack_name))
]
if not option.override_all_packs:
check_if_index_is_updated(index_folder_path, content_repo,
current_commit_hash, previous_commit_hash,
storage_bucket)
# google cloud bigquery client initialized
bq_client = init_bigquery_client(service_account)
packs_statistic_df = get_packs_statistics_dataframe(bq_client)
updated_private_packs_ids = []
if private_bucket_name: # Add private packs to the index
private_storage_bucket = storage_client.bucket(private_bucket_name)
private_packs, _, _, updated_private_packs_ids = update_index_with_priced_packs(
private_storage_bucket, extract_destination_path,
index_folder_path, pack_names)
else: # skipping private packs
logging.debug("Skipping index update of priced packs")
private_packs = []
# clean index and gcs from non existing or invalid packs
clean_non_existing_packs(index_folder_path, private_packs, storage_bucket)
# starting iteration over packs
for pack in packs_list:
task_status, user_metadata = pack.load_user_metadata()
if not task_status:
pack.status = PackStatus.FAILED_LOADING_USER_METADATA.value
pack.cleanup()
continue
task_status, pack_content_items = pack.collect_content_items()
if not task_status:
pack.status = PackStatus.FAILED_COLLECT_ITEMS.name
pack.cleanup()
continue
task_status, integration_images = pack.upload_integration_images(
storage_bucket)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
continue
task_status, author_image = pack.upload_author_image(storage_bucket)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
continue
task_status, pack_was_modified = pack.detect_modified(
content_repo, index_folder_path, current_commit_hash,
previous_commit_hash)
if not task_status:
pack.status = PackStatus.FAILED_DETECTING_MODIFIED_FILES.name
pack.cleanup()
continue
task_status = pack.format_metadata(
user_metadata=user_metadata,
pack_content_items=pack_content_items,
integration_images=integration_images,
author_image=author_image,
index_folder_path=index_folder_path,
packs_dependencies_mapping=packs_dependencies_mapping,
build_number=build_number,
commit_hash=current_commit_hash,
packs_statistic_df=packs_statistic_df,
pack_was_modified=pack_was_modified)
if not task_status:
pack.status = PackStatus.FAILED_METADATA_PARSING.name
pack.cleanup()
continue
task_status, not_updated_build = pack.prepare_release_notes(
index_folder_path, build_number, pack_was_modified)
if not task_status:
pack.status = PackStatus.FAILED_RELEASE_NOTES.name
pack.cleanup()
continue
if not_updated_build:
pack.status = PackStatus.PACK_IS_NOT_UPDATED_IN_RUNNING_BUILD.name
pack.cleanup()
continue
task_status = pack.remove_unwanted_files(remove_test_playbooks)
if not task_status:
pack.status = PackStatus.FAILED_REMOVING_PACK_SKIPPED_FOLDERS
pack.cleanup()
continue
task_status = pack.sign_pack(signature_key)
if not task_status:
pack.status = PackStatus.FAILED_SIGNING_PACKS.name
pack.cleanup()
continue
task_status, zip_pack_path = pack.zip_pack()
if not task_status:
pack.status = PackStatus.FAILED_ZIPPING_PACK_ARTIFACTS.name
pack.cleanup()
continue
(task_status, skipped_pack_uploading, full_pack_path) = \
pack.upload_to_storage(zip_pack_path, pack.latest_version,
storage_bucket, override_all_packs
or pack_was_modified)
if not task_status:
pack.status = PackStatus.FAILED_UPLOADING_PACK.name
pack.cleanup()
continue
task_status, exists_in_index = pack.check_if_exists_in_index(
index_folder_path)
if not task_status:
pack.status = PackStatus.FAILED_SEARCHING_PACK_IN_INDEX.name
pack.cleanup()
continue
task_status = pack.prepare_for_index_upload()
if not task_status:
pack.status = PackStatus.FAILED_PREPARING_INDEX_FOLDER.name
pack.cleanup()
continue
task_status = update_index_folder(index_folder_path=index_folder_path,
pack_name=pack.name,
pack_path=pack.path,
pack_version=pack.latest_version,
hidden_pack=pack.hidden)
if not task_status:
pack.status = PackStatus.FAILED_UPDATING_INDEX_FOLDER.name
pack.cleanup()
continue
# in case that pack already exist at cloud storage path and in index, don't show that the pack was changed
if skipped_pack_uploading and exists_in_index:
pack.status = PackStatus.PACK_ALREADY_EXISTS.name
pack.cleanup()
continue
pack.status = PackStatus.SUCCESS.name
# upload core packs json to bucket
upload_core_packs_config(storage_bucket, build_number, index_folder_path)
# finished iteration over content packs
upload_index_to_storage(index_folder_path=index_folder_path,
extract_destination_path=extract_destination_path,
index_blob=index_blob,
build_number=build_number,
private_packs=private_packs,
current_commit_hash=current_commit_hash,
index_generation=index_generation,
force_upload=force_upload,
previous_commit_hash=previous_commit_hash)
# upload id_set.json to bucket
upload_id_set(storage_bucket, id_set_path)
# get the lists of packs divided by their status
successful_packs, skipped_packs, failed_packs = get_packs_summary(
packs_list)
# Store successful and failed packs list in CircleCI artifacts - to be used in Upload Packs To Marketplace job
packs_results_file_path = os.path.join(
os.path.dirname(packs_artifacts_path),
BucketUploadFlow.PACKS_RESULTS_FILE)
store_successful_and_failed_packs_in_ci_artifacts(
packs_results_file_path, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING,
successful_packs, failed_packs, updated_private_packs_ids)
# summary of packs status
print_packs_summary(successful_packs, skipped_packs, failed_packs,
not is_bucket_upload_flow)
def upload_index_to_storage(index_folder_path: str, extract_destination_path: str, index_blob: Any,
build_number: str, private_packs: list, current_commit_hash: str,
index_generation: int, is_private: bool = False, force_upload: bool = False,
previous_commit_hash: str = None, landing_page_sections: dict = None):
"""
Upload updated index zip to cloud storage.
:param index_folder_path: index folder full path.
:param extract_destination_path: extract folder full path.
:param index_blob: google cloud storage object that represents index.zip blob.
:param build_number: circleCI build number, used as an index revision.
:param private_packs: List of private packs and their price.
:param current_commit_hash: last commit hash of head.
:param index_generation: downloaded index generation.
:param is_private: Indicates if upload is private.
:param force_upload: Indicates if force upload or not.
:param previous_commit_hash: The previous commit hash to diff with.
:param landing_page_sections: landingPage sections.
:returns None.
"""
if force_upload:
# If we force upload we don't want to update the commit in the index.json file,
# this is to be able to identify all changed packs in the next upload
commit = previous_commit_hash
logging.info('Force upload flow - Index commit hash shuould not be changed')
else:
# Otherwise, update the index with the current commit hash (the commit of the upload)
commit = current_commit_hash
logging.info('Updating production index commit hash to master last commit hash')
if not landing_page_sections:
landing_page_sections = load_json(LANDING_PAGE_SECTIONS_PATH)
logging.debug(f'commit hash is: {commit}')
with open(os.path.join(index_folder_path, f"{GCPConfig.INDEX_NAME}.json"), "w+") as index_file:
index = {
'revision': build_number,
'modified': datetime.utcnow().strftime(Metadata.DATE_FORMAT),
'packs': private_packs,
'commit': commit,
'landingPage': {'sections': landing_page_sections.get('sections', [])}
}
json.dump(index, index_file, indent=4)
index_zip_name = os.path.basename(index_folder_path)
index_zip_path = shutil.make_archive(base_name=index_folder_path, format="zip",
root_dir=extract_destination_path, base_dir=index_zip_name)
try:
index_blob.reload()
current_index_generation = index_blob.generation
index_blob.cache_control = "no-cache,max-age=0" # disabling caching for index blob
if is_private or current_index_generation == index_generation:
index_blob.upload_from_filename(index_zip_path)
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage.")
else:
logging.critical(f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation")
logging.critical(f"Downloaded index generation: {index_generation}")
logging.critical(f"Current index generation: {current_index_generation}")
sys.exit(0)
except Exception:
logging.exception(f"Failed in uploading {GCPConfig.INDEX_NAME}.")
sys.exit(1)
finally:
shutil.rmtree(index_folder_path)
