def select_packs_for_calculation() -> list:
"""
Select the packs on which the dependencies will be calculated on
Returns:
A list of packs
"""
IGNORED_FILES.append(GCPConfig.BASE_PACK) # skip dependency calculation of Base pack
packs = []
for pack in os.scandir(PACKS_FULL_PATH):
if not pack.is_dir() or pack.name in IGNORED_FILES:
logging.warning(f"Skipping dependency calculation of {pack.name} pack.")
continue # skipping ignored packs
packs.append(pack.name)
return packs
def is_pack_hidden(pack_id: str) -> bool:
"""
Check if the given pack is deprecated.
:param pack_id: ID of the pack.
:return: True if the pack is deprecated, i.e. has 'hidden: true' field, False otherwise.
"""
metadata_path = os.path.join(PACKS_FULL_PATH, pack_id, PACK_METADATA_FILE)
if pack_id and os.path.isfile(metadata_path):
with open(metadata_path, 'r') as json_file:
pack_metadata = json.load(json_file)
return pack_metadata.get('hidden', False)
else:
logging.warning(f'Could not open metadata file of pack {pack_id}')
return False
def check_if_need_to_upload(pc_successful_packs_dict: dict, pc_failed_packs_dict: dict,
pc_successful_private_packs_dict: dict, pc_uploaded_images: dict):
""" If the three dicts are empty then no upload was done in Prepare Content step, so we need to skip uploading
Args:
pc_successful_packs_dict: The successful packs dict
pc_failed_packs_dict: The failed packs dict
pc_successful_private_packs_dict : The successful private packs dict
pc_uploaded_images: The image data dict
"""
if not pc_successful_packs_dict and not pc_failed_packs_dict and not pc_successful_private_packs_dict and not \
pc_uploaded_images:
logging.warning("Production bucket is updated with origin/master.")
logging.warning("Skipping Upload To Marketplace Storage Step.")
sys.exit(0)
def install_nightly_packs(client: demisto_client,
host: str,
packs_to_install: List,
request_timeout: int = 999999):
"""
Install content packs on nightly build.
We will catch the exception if pack fails to install and send the request to install packs again without the
corrupted pack.
Args:
client(demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
Returns:
None: No data returned.
"""
logging.info(f'Installing packs on server {host}')
# make the pack installation request
all_packs_install_successfully = False
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
while not all_packs_install_successfully:
try:
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs in server {host}:\n{packs_to_install_str}')
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs on server {host}- with status code {status_code}\n{message}\n')
break
except Exception as e:
all_packs_install_successfully = False
malformed_pack_id = find_malformed_pack_id(str(e))
if not malformed_pack_id:
logging.exception('The request to install packs has failed')
raise
pack_ids_to_install = {pack['id'] for pack in packs_to_install}
malformed_pack_id = malformed_pack_id[0]
if malformed_pack_id not in pack_ids_to_install:
logging.exception(
f'The pack {malformed_pack_id} has failed to install even though it was not in the installation list')
raise
logging.warning(f'The request to install packs on server {host} has failed, retrying without {malformed_pack_id}')
# Remove the malformed pack from the pack to install list.
packs_to_install = [pack for pack in packs_to_install if pack['id'] not in malformed_pack_id]
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
def test_instances(secret_conf_path, server, username, password):
integrations = get_integrations(secret_conf_path)
instance_ids = []
failed_integrations = []
integrations_counter = 0
content_installation_client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
install_new_content(content_installation_client, server)
for integration in integrations:
c = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
integrations_counter += 1
integration_name = integration.get('name')
integration_instance_name = integration.get('instance_name', '')
integration_params = integration.get('params')
devops_comments = integration.get('devops_comments')
product_description = integration.get('product_description', '')
is_byoi = integration.get('byoi', True)
has_integration = integration.get('has_integration', True)
validate_test = integration.get('validate_test', True)
if has_integration:
try:
instance_id, failure_message = __create_integration_instance(
server,
username,
password,
integration_name,
integration_instance_name,
integration_params,
is_byoi,
validate_test=validate_test)
except Exception:
logging.exception(
f'Failed to configure integration with name {integration_name}'
)
failed_integrations.append(
f"{integration_name} {product_description} - devops comments: {devops_comments}"
)
continue
if failure_message == 'No configuration':
logging.warning(
f"skipping {integration_name} as it exists in content-test-conf conf.json but not in content repo"
)
continue
if not instance_id:
logging.error(
f'Failed to create instance of {integration_name} with message: {failure_message}'
)
failed_integrations.append(
"{} {} - devops comments: {}".format(
integration_name, product_description,
devops_comments))
else:
instance_ids.append(instance_id)
logging.success(
f'Create integration {integration_name} succeed')
__delete_integrations_instances(c, instance_ids)
return failed_integrations, integrations_counter
def main():
install_logging('Copy_and_Upload_Packs.log', logger=logging)
options = options_handler()
packs_artifacts_path = options.artifacts_path
extract_destination_path = options.extract_path
production_bucket_name = options.production_bucket_name
build_bucket_name = options.build_bucket_name
service_account = options.service_account
build_number = options.ci_build_number
circle_branch = options.circle_branch
production_base_path = options.production_base_path
target_packs = options.pack_names
marketplace = options.marketplace
# Google cloud storage client initialized
storage_client = init_storage_client(service_account)
production_bucket = storage_client.bucket(production_bucket_name)
build_bucket = storage_client.bucket(build_bucket_name)
# Initialize build and prod base paths
build_bucket_path = os.path.join(GCPConfig.BUILD_PATH_PREFIX, circle_branch, build_number, marketplace)
build_bucket_base_path = os.path.join(build_bucket_path, GCPConfig.CONTENT_PACKS_PATH)
# Relevant when triggering test upload flow
if production_bucket_name:
GCPConfig.PRODUCTION_BUCKET = production_bucket_name
# Download and extract build index from build and prod buckets
build_index_folder_path, build_index_blob, build_index_generation = \
download_and_extract_index(build_bucket, extract_destination_path, build_bucket_base_path)
# Get the successful and failed packs file from Prepare Content step in Create Instances job if there are
packs_results_file_path = os.path.join(os.path.dirname(packs_artifacts_path), BucketUploadFlow.PACKS_RESULTS_FILE)
pc_successful_packs_dict, pc_failed_packs_dict, pc_successful_private_packs_dict, \
pc_uploaded_images = get_upload_data(packs_results_file_path, BucketUploadFlow.PREPARE_CONTENT_FOR_TESTING)
logging.debug(f"Successful packs from Prepare Content: {pc_successful_packs_dict}")
logging.debug(f"Failed packs from Prepare Content: {pc_failed_packs_dict}")
logging.debug(f"Successful private packs from Prepare Content: {pc_successful_private_packs_dict}")
logging.debug(f"Images from Prepare Content: {pc_uploaded_images}")
# Check if needs to upload or not
check_if_need_to_upload(pc_successful_packs_dict, pc_failed_packs_dict, pc_successful_private_packs_dict,
pc_uploaded_images)
# Detect packs to upload
pack_names = get_pack_names(target_packs)
extract_packs_artifacts(packs_artifacts_path, extract_destination_path)
packs_list = [Pack(pack_name, os.path.join(extract_destination_path, pack_name)) for pack_name in pack_names
if os.path.exists(os.path.join(extract_destination_path, pack_name))]
packs_for_current_marketplace = []
for pack in packs_list:
task_status = pack.load_user_metadata()
if not task_status:
pack.status = PackStatus.FAILED_LOADING_USER_METADATA.value
pack.cleanup()
continue
if marketplace not in pack.marketplaces:
logging.warning(f"Skipping {pack.name} pack as it is not supported in the current marketplace.")
pack.status = PackStatus.NOT_RELEVANT_FOR_MARKETPLACE.name
pack.cleanup()
continue
packs_for_current_marketplace.append(pack)
# Starting iteration over packs
for pack in packs_for_current_marketplace:
# Indicates whether a pack has failed to upload on Prepare Content step
task_status, pack_status = pack.is_failed_to_upload(pc_failed_packs_dict)
if task_status:
pack.status = pack_status
pack.cleanup()
continue
task_status = pack.copy_integration_images(
production_bucket, build_bucket, pc_uploaded_images, production_base_path, build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_IMAGES_UPLOAD.name
pack.cleanup()
continue
task_status = pack.copy_author_image(
production_bucket, build_bucket, pc_uploaded_images, production_base_path, build_bucket_base_path)
if not task_status:
pack.status = PackStatus.FAILED_AUTHOR_IMAGE_UPLOAD.name
pack.cleanup()
continue
task_status, skipped_pack_uploading = pack.copy_and_upload_to_storage(
production_bucket, build_bucket, pc_successful_packs_dict, production_base_path, build_bucket_base_path)
if skipped_pack_uploading:
pack.status = PackStatus.PACK_ALREADY_EXISTS.name
pack.cleanup()
continue
if not task_status:
pack.status = PackStatus.FAILED_UPLOADING_PACK.name
pack.cleanup()
continue
pack.status = PackStatus.SUCCESS.name
# upload core packs json to bucket
upload_core_packs_config(production_bucket, build_number, extract_destination_path, build_bucket,
production_base_path, build_bucket_base_path)
# finished iteration over content packs
copy_index(build_index_folder_path, build_index_blob, build_index_generation, production_bucket,
build_bucket, production_base_path, build_bucket_base_path)
# upload id_set.json to bucket
copy_id_set(production_bucket, build_bucket, production_base_path, build_bucket_base_path)
# get the lists of packs divided by their status
successful_packs, skipped_packs, failed_packs = get_packs_summary(packs_list)
# Store successful and failed packs list in CircleCI artifacts
store_successful_and_failed_packs_in_ci_artifacts(
packs_results_file_path, BucketUploadFlow.UPLOAD_PACKS_TO_MARKETPLACE_STORAGE, successful_packs, failed_packs,
list(pc_successful_private_packs_dict)
)
# verify that the successful from Prepare content and are the ones that were copied
verify_copy(successful_packs, pc_successful_packs_dict)
# summary of packs status
print_packs_summary(successful_packs, skipped_packs, failed_packs)
def run_private_test_scenario(
tests_settings: SettingsTester, t: dict, default_test_timeout: int,
skipped_tests_conf: dict, nightly_integrations: list,
skipped_integrations_conf: set, skipped_integration: set,
filtered_tests: list, skipped_tests: set, secret_params: dict,
failed_playbooks: list, playbook_skipped_integration: set,
succeed_playbooks: list, slack: str, circle_ci: str, build_number: str,
server: str, build_name: str, server_numeric_version: str,
demisto_user: str, demisto_pass: str, demisto_api_key: str):
"""
Checks to see if test should run given the scenario. If the test should run, it will collect the
integrations which are required to run the test.
:param tests_settings: SettingsTester object which contains the test variables
:param t: Options being passed to the test. PID, Docker Threshold, Timeout, etc.
:param default_test_timeout: Time in seconds indicating when the test should timeout if no
status is reported.
:param skipped_tests_conf: Collection of the tests which are skipped.
:param nightly_integrations: List of integrations which should only be tested on a nightly build.
:param skipped_integrations_conf: Collection of integrations which are skiped.
:param skipped_integration: Set of skipped integrations. Currently not used in private.
:param filtered_tests: List of tests excluded from testing.
:param skipped_tests: List of skipped tests.
:param secret_params: Parameters found in the content-test-conf. Used to configure the instance.
:param failed_playbooks: List of failed playbooks, additional failed playbooks will be added if
they failed.
:param playbook_skipped_integration: Not used.
:param succeed_playbooks: List of playbooks which have passed tests.
:param slack: Slack client used for notifications.
:param circle_ci: CircleCI token. Used to get name of dev who triggered the build.
:param build_number: The build number of the CI run. Used in slack message.
:param server: The FQDN of the server tests are being ran on.
:param build_name: Name of the build. (Nightly, etc.)
:param server_numeric_version: Version of XSOAR currently installed on the server.
:param demisto_user: Username of the demisto user running the tests.
:param demisto_pass: Password of the demisto user running the tests.
:param demisto_api_key: API key for the demisto instance.
:return:
"""
playbook_id = t['playbookID']
integrations_conf = t.get('integrations', [])
instance_names_conf = t.get('instance_names', [])
test_message = 'playbook: ' + playbook_id
test_options = {'timeout': t.get('timeout', default_test_timeout)}
if not isinstance(integrations_conf, list):
integrations_conf = [
integrations_conf,
]
if not isinstance(instance_names_conf, list):
instance_names_conf = [
instance_names_conf,
]
test_skipped_integration, integrations, is_nightly_integration = collect_integrations(
integrations_conf, skipped_integration, skipped_integrations_conf,
nightly_integrations)
if playbook_id in filtered_tests:
playbook_skipped_integration.update(test_skipped_integration)
# Skip tests that are missing from filtered list
if filtered_tests and playbook_id not in filtered_tests:
return
# Skip bad test
if playbook_id in skipped_tests_conf:
skipped_tests.add(
f'{playbook_id} - reason: {skipped_tests_conf[playbook_id]}')
return
# Skip integration
if test_skipped_integration:
return
# Skip version mismatch test
test_from_version = t.get('fromversion', '0.0.0')
test_to_version = t.get('toversion', '99.99.99')
if not LooseVersion(test_from_version) <= LooseVersion(
server_numeric_version) <= LooseVersion(test_to_version):
warning_message = f'Test {test_message} ignored due to version mismatch ' \
f'(test versions: {test_from_version}-{test_to_version})'
logging.warning(warning_message)
return
placeholders_map = {'%%SERVER_HOST%%': server}
are_params_set = set_integration_params(demisto_api_key, integrations,
secret_params, instance_names_conf,
playbook_id, placeholders_map)
if not are_params_set:
failed_playbooks.append(playbook_id)
return
test_message = update_test_msg(integrations, test_message)
run_test(tests_settings, demisto_user, demisto_pass, failed_playbooks,
integrations, playbook_id, succeed_playbooks, test_message,
test_options, slack, circle_ci, build_number, server, build_name)
if malformed_ids := find_malformed_pack_id(ex.body):
raise MalformedPackException(malformed_ids)
if 'Item not found' in ex.body:
raise GeneralItemNotFoundError(ex.body)
raise ex
try:
logging.info(f'Installing packs on server {host}')
try:
call_install_packs_request(packs_to_install)
except MalformedPackException as e:
# if this is malformed pack error, remove malformed packs and retry until success
handle_malformed_pack_ids(e.malformed_ids, packs_to_install)
logging.warning(
f'The request to install packs on server {host} has failed, retrying without packs '
f'{e.malformed_ids}')
return install_packs(client, host, [
pack for pack in packs_to_install
if pack['id'] not in e.malformed_ids
], request_timeout)
except GCPTimeOutException as e:
# if this is a gcp timeout, try only once more
logging.warning(
f'The request to install packs on server {host} has failed due to timeout awaiting response'
f' headers while trying to install pack {e.pack_id}, trying again for one more time'
)
call_install_packs_request(packs_to_install)
except GeneralItemNotFoundError as e: