def bucket_upload_results():
steps_fields = []
logging.info(f'retrieving upload data from "{PACK_RESULTS_PATH}"')
successful_packs, failed_packs, successful_private_packs, _ = get_upload_data(
PACK_RESULTS_PATH, BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE
)
if successful_packs:
steps_fields += [{
"title": "Successful Packs:",
"value": "\n".join(sorted([pack_name for pack_name in {*successful_packs}], key=lambda s: s.lower())),
"short": False
}]
if failed_packs:
steps_fields += [{
"title": "Failed Packs:",
"value": "\n".join(sorted([pack_name for pack_name in {*failed_packs}], key=lambda s: s.lower())),
"short": False
}]
if successful_private_packs:
steps_fields += [{
"title": "Successful Private Packs:",
"value": "\n".join(sorted([pack_name for pack_name in {*successful_private_packs}],
key=lambda s: s.lower())),
"short": False
}]
return steps_fields
def bucket_upload_results(bucket_artifact_folder):
steps_fields = []
pack_results_path = os.path.join(bucket_artifact_folder,
BucketUploadFlow.PACKS_RESULTS_FILE)
marketplace_name = os.path.basename(bucket_artifact_folder).upper()
logging.info(f'retrieving upload data from "{pack_results_path}"')
successful_packs, failed_packs, successful_private_packs, _ = get_upload_data(
pack_results_path,
BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE)
if successful_packs:
steps_fields += [{
'title':
f'Successful {marketplace_name} Packs:',
'value':
'\n'.join(
sorted([pack_name for pack_name in {*successful_packs}],
key=lambda s: s.lower())),
'short':
False
}]
if failed_packs:
steps_fields += [{
'title':
f'Failed {marketplace_name} Packs:',
'value':
'\n'.join(
sorted([pack_name for pack_name in {*failed_packs}],
key=lambda s: s.lower())),
'short':
False
}]
if successful_private_packs:
# No need to indicate the marketplace name as private packs only upload to xsoar marketplace.
steps_fields += [{
'title':
'Successful Private Packs:',
'value':
'\n'.join(
sorted(
[pack_name for pack_name in {*successful_private_packs}],
key=lambda s: s.lower())),
'short':
False
}]
return steps_fields
def get_attachments_for_bucket_upload_flow(build_url,
job_name,
packs_results_file_path=None):
steps_fields = get_entities_fields(entity_title="Failed Steps")
color = 'good' if not steps_fields else 'danger'
title = f'{BucketUploadFlow.BUCKET_UPLOAD_BUILD_TITLE} - Success' if not steps_fields \
else f'{BucketUploadFlow.BUCKET_UPLOAD_BUILD_TITLE} - Failure'
if job_name and color == 'danger':
steps_fields = [{
"title": f'Job Failed: {job_name}',
"value": '',
"short": False
}] + steps_fields
if job_name and job_name == BucketUploadFlow.UPLOAD_JOB_NAME:
successful_packs, failed_packs, successful_private_packs, _ = get_upload_data(
packs_results_file_path,
BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE)
if successful_packs:
steps_fields += [{
"title":
"Successful Packs:",
"value":
"\n".join(
sorted([pack_name for pack_name in {*successful_packs}],
key=lambda s: s.lower())),
"short":
False
}]
if failed_packs:
steps_fields += [{
"title":
"Failed Packs:",
"value":
"\n".join(
sorted([pack_name for pack_name in {*failed_packs}],
key=lambda s: s.lower())),
"short":
False
}]
if successful_private_packs:
steps_fields += [{
"title":
"Successful Private Packs:",
"value":
"\n".join(
sorted([
pack_name for pack_name in {*successful_private_packs}
],
key=lambda s: s.lower())),
"short":
False
}]
if job_name and job_name != 'Upload Packs To Marketplace' and color == 'good':
logging.info(
'On bucket upload flow we are not notifying on jobs that are not Upload Packs. exiting...'
)
sys.exit(0)
container_build_url = build_url + '#queue-placeholder/containers/0'
content_team_attachment = [{
'fallback': title,
'color': color,
'title': title,
'title_link': container_build_url,
'fields': steps_fields
}]
return content_team_attachment
def main():
install_logging('Copy_and_Upload_Packs.log', logger=logging)
options = options_handler()
packs_artifacts_path = options.artifacts_path
extract_destination_path = options.extract_path
production_bucket_name = options.production_bucket_name
build_bucket_name = options.build_bucket_name
service_account = options.service_account
build_number = options.ci_build_number
circle_branch = options.circle_branch
production_base_path = options.production_base_path
target_packs = options.pack_names
marketplace = options.marketplace
# Google cloud storage client initialized
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(production_bucket_name)
build_bucket = storage_client.bucket(build_bucket_name)
# Initialize build and prod base paths
build_bucket_path = os.path.join(GCPConfig.BUILD_PATH_PREFIX,
circle_branch, build_number, marketplace)
build_bucket_base_path = os.path.join(build_bucket_path,
GCPConfig.CONTENT_PACKS_PATH)
# Relevant when triggering test upload flow
if production_bucket_name:
GCPConfig.PRODUCTION_BUCKET = production_bucket_name
# Download and extract build index from build and prod buckets
build_index_folder_path, build_index_blob, build_index_generation = \
download_and_extract_index(build_bucket, extract_destination_path, build_bucket_base_path)
# Get the successful and failed packs file from Prepare Content step in Create Instances job if there are
packs_results_file_path = os.path.join(
os.path.dirname(packs_artifacts_path),
BucketUploadFlow.PACKS_RESULTS_FILE)
pc_successful_packs_dict, pc_failed_packs_dict, pc_successful_private_packs_dict, \
pc_uploaded_images = get_upload_data(packs_results_file_path, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
logging.debug(
f"Successful packs from Prepare Content: {pc_successful_packs_dict}")
logging.debug(f"Failed packs from Prepare Content: {pc_failed_packs_dict}")
logging.debug(
f"Successful private packs from Prepare Content: {pc_successful_private_packs_dict}"
)
logging.debug(f"Images from Prepare Content: {pc_uploaded_images}")
# Check if needs to upload or not
check_if_need_to_upload(pc_successful_packs_dict, pc_failed_packs_dict,
pc_successful_private_packs_dict,
pc_uploaded_images)
# Detect packs to upload
pack_names = get_pack_names(target_packs)
extract_packs_artifacts(packs_artifacts_path, extract_destination_path)
packs_list = [
Pack(pack_name, os.path.join(extract_destination_path, pack_name),
marketplace) for pack_name in pack_names
if os.path.exists(os.path.join(extract_destination_path, pack_name))
]
# Starting iteration over packs
for pack in packs_list:
# Indicates whether a pack has failed to upload on Prepare Content step
task_status, pack_status = pack.is_failed_to_upload(
pc_failed_packs_dict)
if task_status:
pack.status = pack_status
pack.cleanup()
continue
task_status = pack.load_user_metadata(marketplace)
if not task_status:
pack.status = PackStatus.FAILED_LOADING_USER_METADATA.name
pack.cleanup()
continue
task_status = pack.copy_integration_images(production_bucket,
build_bucket,
pc_uploaded_images,
production_base_path,
build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
continue
task_status = pack.copy_author_image(production_bucket, build_bucket,
pc_uploaded_images,
production_base_path,
build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
continue
task_status, skipped_pack_uploading = pack.copy_and_upload_to_storage(
production_bucket, build_bucket, pc_successful_packs_dict,
production_base_path, build_bucket_base_path)
if skipped_pack_uploading:
pack.status = PackStatus.PACK_ALREADY_EXISTS.name
pack.cleanup()
continue
if not task_status:
pack.status = PackStatus.FAILED_UPLOADING_PACK.name
pack.cleanup()
continue
pack.status = PackStatus.SUCCESS.name
# upload core packs json to bucket
upload_core_packs_config(production_bucket, build_number,
extract_destination_path, build_bucket,
production_base_path, build_bucket_base_path)
# finished iteration over content packs
copy_index(build_index_folder_path, build_index_blob,
build_index_generation, production_bucket, build_bucket,
production_base_path, build_bucket_base_path)
# upload id_set.json to bucket
copy_id_set(production_bucket, build_bucket, production_base_path,
build_bucket_base_path)
# get the lists of packs divided by their status
successful_packs, skipped_packs, failed_packs = get_packs_summary(
packs_list)
# Store successful and failed packs list in CircleCI artifacts
store_successful_and_failed_packs_in_ci_artifacts(
packs_results_file_path,
BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE, successful_packs,
failed_packs, list(pc_successful_private_packs_dict))
# verify that the successful from Prepare content and are the ones that were copied
verify_copy(successful_packs, pc_successful_packs_dict)
# summary of packs status
print_packs_summary(successful_packs, skipped_packs, failed_packs)
else:
steps_fields = []
color = 'good' if not steps_fields else 'danger'
title = f'{BucketUploadFlow.BUCKET_UPLOAD_BUILD_TITLE} - Success' if not steps_fields \
else f'{BucketUploadFlow.BUCKET_UPLOAD_BUILD_TITLE} - Failure'
if job_name and color == 'danger':
steps_fields = [{
"title": f'Job Failed: {job_name}',
"value": '',
"short": False
}] + steps_fields
if job_name and job_name in BucketUploadFlow.UPLOAD_JOB_NAMES:
successful_packs, failed_packs, successful_private_packs, _ = get_upload_data(
packs_results_file_path, BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE
)
if successful_packs:
steps_fields += [{
"title": "Successful Packs:",
"value": "\n".join(sorted([pack_name for pack_name in {*successful_packs}], key=lambda s: s.lower())),
"short": False
}]
if failed_packs:
steps_fields += [{
"title": "Failed Packs:",
"value": "\n".join(sorted([pack_name for pack_name in {*failed_packs}], key=lambda s: s.lower())),
"short": False
}]
if successful_private_packs:
steps_fields += [{
