def main():
""" Main function for iterating over existing packs folder in content repo and creating json of all
packs dependencies. The logic of pack dependency is identical to sdk find-dependencies command.
"""
install_logging('Calculate_Packs_Dependencies.log', include_process_name=True, logger=logging)
option = option_handler()
output_path = option.output_path
id_set_path = option.id_set_path
id_set = get_id_set(id_set_path)
pack_dependencies_result: dict = {}
logging.info("Selecting packs for dependencies calculation")
packs = select_packs_for_calculation()
calculate_all_packs_dependencies(pack_dependencies_result, id_set, packs)
logging.info(f"Number of created pack dependencies entries: {len(pack_dependencies_result.keys())}")
# finished iteration over pack folders
logging.success("Finished dependencies calculation")
with open(output_path, 'w') as pack_dependencies_file:
json.dump(pack_dependencies_result, pack_dependencies_file, indent=4)
logging.success(f"Created packs dependencies file at: {output_path}")
def xsiam_configure_and_install_flow(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
logging.info('Retrieving the credentials for Cortex XSIAM server')
xsiam_machine = options.xsiam_machine
xsiam_servers = get_json_file(options.xsiam_servers_path)
api_key, server_numeric_version, base_url, xdr_auth_id = XSIAMBuild.get_xsiam_configuration(
xsiam_machine, xsiam_servers)
# Configure the Server
server = XSIAMServer(api_key, server_numeric_version, base_url,
xdr_auth_id, xsiam_machine)
XSIAMBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
# Acquire the server's host and install new uploaded content packs
install_packs_from_content_packs_to_install_path(
[server], options.pack_ids_to_install, server.name)
logging.success(
f'Finished installing all content packs in {xsiam_machine}')
def copy_id_set(production_bucket: Bucket, build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the id_set.json artifact from the build bucket to the production bucket.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied from.
storage_base_path (str): the path to upload the id_set.json to.
build_bucket_base_path (str): the path in the build bucket of the id_set.json.
"""
build_id_set_path = os.path.join(os.path.dirname(build_bucket_base_path), 'id_set.json')
build_id_set_blob = build_bucket.blob(build_id_set_path)
if not build_id_set_blob.exists():
logging.error(f"id_set.json file does not exists in build bucket in path: {build_id_set_path}")
sys.exit(1)
prod_id_set_path = os.path.join(os.path.dirname(storage_base_path), 'id_set.json')
try:
copied_blob = build_bucket.copy_blob(
blob=build_id_set_blob, destination_bucket=production_bucket, new_name=prod_id_set_path
)
if not copied_blob.exists():
logging.error(f"Failed to upload id_set.json to {prod_id_set_path}")
sys.exit(1)
else:
logging.success("Finished uploading id_set.json to storage.")
except Exception as e:
logging.exception(f"Failed copying ID Set. Additional Info: {str(e)}")
sys.exit(1)
def download_and_extract_index(build_bucket: Bucket,
extract_destination_path: str,
build_bucket_base_path: str):
"""Downloads and extracts production and build indexes zip from cloud storage.
Args:
build_bucket (google.cloud.storage.bucket.Bucket): google storage bucket where build index.zip is stored.
extract_destination_path (str): the full path of extract folder.
build_bucket_base_path (str): the path in the build bucket of the index.
Returns:
str: extracted build index folder full path.
Blob: google cloud storage object that represents prod index.zip blob.
Blob: google cloud storage object that represents build index.zip blob.
str: downloaded prod index generation.
str: downloaded build index generation.
"""
build_index_storage_path = os.path.join(build_bucket_base_path,
f"{GCPConfig.INDEX_NAME}.zip")
download_build_index_path = os.path.join(extract_destination_path,
f"{GCPConfig.INDEX_NAME}.zip")
build_index_blob = build_bucket.blob(build_index_storage_path)
build_index_folder_path = os.path.join(extract_destination_path,
GCPConfig.INDEX_NAME)
if not os.path.exists(extract_destination_path):
os.mkdir(extract_destination_path)
if not build_index_blob.exists():
logging.error(
f"No build index was found in path: {build_index_storage_path}")
sys.exit(1)
build_index_blob.reload()
build_index_generation = build_index_blob.generation
build_index_blob.download_to_filename(
download_build_index_path, if_generation_match=build_index_generation)
if os.path.exists(download_build_index_path):
with ZipFile(download_build_index_path, 'r') as index_zip:
index_zip.extractall(extract_destination_path)
if not os.path.exists(build_index_folder_path):
logging.error(
f"Failed creating build {GCPConfig.INDEX_NAME} folder with extracted data."
)
sys.exit(1)
os.remove(download_build_index_path)
logging.success(
f"Finished downloading and extracting build {GCPConfig.INDEX_NAME} file to "
f"{extract_destination_path}")
return build_index_folder_path, build_index_blob, build_index_generation
else:
logging.error(
f"Failed to download build {GCPConfig.INDEX_NAME}.zip file from cloud storage."
)
sys.exit(1)
def call_install_packs_request(packs):
try:
logging.debug(
f'Installing the following packs on server {host}:\n{[pack["id"] for pack in packs]}'
)
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/marketplace/install',
method='POST',
body={
'packs': packs,
'ignoreWarnings': True
},
accept='application/json',
_request_timeout=request_timeout)
if status_code in range(200, 300) and status_code != 204:
packs_data = [{
'ID': pack.get('id'),
'CurrentVersion': pack.get('currentVersion')
} for pack in ast.literal_eval(response_data)]
logging.success(
f'Packs were successfully installed on server {host}')
logging.debug(
f'The packs that were successfully installed on server {host}:\n{packs_data}'
)
except ApiException as ex:
if 'timeout awaiting response' in ex.body:
raise GCPTimeOutException(ex.body)
if malformed_ids := find_malformed_pack_id(ex.body):
raise MalformedPackException(malformed_ids)
if 'Item not found' in ex.body:
raise GeneralItemNotFoundError(ex.body)
raise ex
def copy_index(index_folder_path: str, build_index_blob: Blob, build_index_generation: str, production_bucket: Bucket,
build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the build bucket index to the production bucket index path.
Args:
index_folder_path (str): index folder full path.
build_index_blob (Blob): google cloud storage object that represents build index.zip blob.
build_index_generation (str): downloaded build index generation.
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied from.
storage_base_path (str): the path to upload the index to.
build_bucket_base_path (str): the path in the build bucket of the index.
"""
try:
build_index_blob.reload()
build_current_index_generation = build_index_blob.generation
# disabling caching for prod index blob
prod_index_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.zip")
prod_index_blob = production_bucket.blob(prod_index_storage_path)
prod_index_blob.cache_control = "no-cache,max-age=0"
prod_index_json_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.json")
prod_index_json_blob = production_bucket.blob(prod_index_json_storage_path)
prod_index_json_blob.cache_control = "no-cache,max-age=0"
if build_current_index_generation == build_index_generation:
copied_index = build_bucket.copy_blob(
blob=build_index_blob, destination_bucket=production_bucket, new_name=prod_index_storage_path
)
if copied_index.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage.")
else:
logging.error("Failed copying index.zip from build index - blob does not exist.")
sys.exit(1)
copied_index_json_blob = build_bucket.blob(
os.path.join(build_bucket_base_path, f"{GCPConfig.INDEX_NAME}.json")
)
copied_index_json = build_bucket.copy_blob(
blob=copied_index_json_blob, destination_bucket=production_bucket, new_name=prod_index_json_storage_path
)
if copied_index_json.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.json to storage.")
else:
logging.error("Failed copying index.json from build index - blob does not exist.")
sys.exit(1)
else:
logging.error(f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation")
logging.error(f"Downloaded build index generation: {build_index_generation}")
logging.error(f"Current build index generation: {build_current_index_generation}")
sys.exit(1)
except Exception as e:
logging.exception(f"Failed copying {GCPConfig.INDEX_NAME}. Additional Info: {str(e)}")
sys.exit(1)
finally:
shutil.rmtree(index_folder_path)
def run_test_logic(tests_settings: Any, c: Any, failed_playbooks: list,
integrations: list, playbook_id: str,
succeed_playbooks: list, test_message: str,
test_options: dict, slack: Any, circle_ci: str,
build_number: str, server_url: str, demisto_user: str,
demisto_pass: str, build_name: str) -> bool:
"""
run_test_logic handles the testing of the integration by triggering check_integration. afterwards
it will check the status of the test and report success or add the failed test to the list of
failed integrations.
:param tests_settings: SettingsTester object which contains the test variables
:param c: Client for connecting to XSOAR via demisto-py
:param failed_playbooks: List of failed playbooks, additional failed playbooks will be added if
they failed.
:param integrations: List of integrations being tested.
:param playbook_id: ID of the test playbook being tested.
:param succeed_playbooks: List of playbooks which have passed tests.
:param test_message: Name of the playbook/integration being tested. This is reported back in the
build and used to print in the console the test being ran.
:param test_options: Options being passed to the test. PID, Docker Threshold, Timeout, etc.
:param slack: Slack client used for notifications.
:param circle_ci: CircleCI token. Used to get name of dev who triggered the build.
:param build_number: The build number of the CI run. Used in slack message.
:param server_url: The FQDN of the server tests are being ran on.
:param demisto_user: Username of the demisto user running the tests.
:param demisto_pass: Password of the demisto user running the tests.
:param build_name: Name of the build. (Nightly, etc.)
:return: Boolean indicating if the test was successful.
"""
status, inc_id = check_integration(c,
server_url,
demisto_user,
demisto_pass,
integrations,
playbook_id,
options=test_options)
if status == PB_Status.COMPLETED:
logging.success(f'PASS: {test_message} succeed')
succeed_playbooks.append(playbook_id)
elif status == PB_Status.NOT_SUPPORTED_VERSION:
logging.info(f'PASS: {test_message} skipped - not supported version')
succeed_playbooks.append(playbook_id)
else:
logging.error(f'Failed: {test_message} failed')
playbook_id_with_mock = playbook_id
playbook_id_with_mock += " (Mock Disabled)"
failed_playbooks.append(playbook_id_with_mock)
succeed = status in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION)
return succeed
def upload_core_packs_config(production_bucket: Bucket, build_number: str, extract_destination_path: str,
build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
"""Uploads the corepacks.json file to the target bucket. This files contains all of the server's core packs, under
the key corepacks, and specifies which core packs should be upgraded upon XSOAR upgrade, under the key upgradeCorePacks.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is uploaded.
build_number (str): CircleCI build number.
extract_destination_path (str): Full path of folder to extract the corepacks file
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where core packs config is downloaded from.
storage_base_path (str): the path to upload the corepacks.json to.
build_bucket_base_path (str): the path in the build bucket of the corepacks.json.
"""
# download the corepacks.json stored in the build bucket to temp dir
build_corepacks_file_path = os.path.join(build_bucket_base_path, GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob = build_bucket.blob(build_corepacks_file_path)
if not build_corepacks_blob.exists():
logging.critical(f"{GCPConfig.CORE_PACK_FILE_NAME} is missing in {build_bucket.name} bucket, exiting...")
sys.exit(1)
temp_corepacks_file_path = os.path.join(extract_destination_path, GCPConfig.CORE_PACK_FILE_NAME)
build_corepacks_blob.download_to_filename(temp_corepacks_file_path)
corepacks_file = load_json(temp_corepacks_file_path)
# change the storage paths to the prod bucket
corepacks_list = corepacks_file.get('corePacks', [])
try:
corepacks_list = [os.path.join(GCPConfig.GCS_PUBLIC_URL, production_bucket.name, storage_base_path,
LATEST_ZIP_REGEX.findall(corepack_path)[0]) for corepack_path in corepacks_list]
except IndexError:
corepacks_list_str = '\n'.join(corepacks_list)
logging.exception(f"GCS paths in build bucket corepacks.json file are not of format: "
f"{GCPConfig.GCS_PUBLIC_URL}/<BUCKET_NAME>/.../content/packs/...\n"
f"List of build bucket corepacks paths:\n{corepacks_list_str}")
sys.exit(1)
# construct core pack data with public gcs urls
core_packs_data = {
'corePacks': corepacks_list,
'upgradeCorePacks': corepacks_file.get('upgradeCorePacks', []),
'buildNumber': build_number
}
# upload core pack json file to gcs
prod_corepacks_file_path = os.path.join(storage_base_path, GCPConfig.CORE_PACK_FILE_NAME)
prod_corepacks_blob = production_bucket.blob(prod_corepacks_file_path)
prod_corepacks_blob.upload_from_string(json.dumps(core_packs_data, indent=4))
logging.success(f"Finished uploading {GCPConfig.CORE_PACK_FILE_NAME} to storage.")
def install_packs(client: demisto_client,
host: str,
packs_to_install: list,
request_timeout: int = 999999,
is_nightly: bool = False):
""" Make a packs installation request.
Args:
client (demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
is_nightly (bool): Is the build nightly or not.
"""
if is_nightly:
install_nightly_packs(client, host, packs_to_install)
return
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
logging.info(f'Installing packs on server {host}')
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs on server {host}:\n{packs_to_install_str}')
# make the pack installation request
try:
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for
pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs - with status code {status_code}\n{message}')
except Exception as e:
logging.exception(f'The request to install packs has failed. Additional info: {str(e)}')
global SUCCESS_FLAG
SUCCESS_FLAG = False
finally:
return SUCCESS_FLAG
def xsoar_configure_and_install_all_packs(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
# Get the host by the ami env
server_to_port_mapping, server_version = XSOARBuild.get_servers(
ami_env=options.ami_env)
logging.info('Retrieving the credentials for Cortex XSOAR server')
secret_conf_file = get_json(file_path=options.secret)
username: str = secret_conf_file.get('username')
password: str = secret_conf_file.get('userPassword')
# Configure the Servers
for server_url, port in server_to_port_mapping.items():
server = XSOARServer(internal_ip=server_url,
port=port,
user_name=username,
password=password)
logging.info(f'Adding Marketplace configuration to {server_url}')
error_msg: str = 'Failed to set marketplace configuration.'
server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION,
error_msg=error_msg)
XSOARBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
# Acquire the server's host and install all content packs (one threaded execution)
logging.info(f'Starting to install all content packs in {server_url}')
server_host: str = server.client.api_client.configuration.host
success_flag = install_all_content_packs_from_build_bucket(
client=server.client,
host=server_host,
server_version=server_version,
bucket_packs_root_path=GCPConfig.
BUILD_BUCKET_PACKS_ROOT_PATH.format(branch=branch_name,
build=build_number,
marketplace='xsoar'),
service_account=options.service_account,
extract_destination_path=options.extract_path)
if success_flag:
logging.success(
f'Finished installing all content packs in {server_url}')
else:
logging.error('Failed to install all packs.')
sys.exit(1)
def main():
install_logging("GetPrivateBuildStatus.log", logger=logging)
if not os.path.isfile(PRIVATE_REPO_WORKFLOW_ID_FILE):
logging.info('Build private repo skipped')
sys.exit(0)
# gets workflow id from the file
with open(PRIVATE_REPO_WORKFLOW_ID_FILE, 'r') as f:
workflow_id = f.read()
# get github_token parameter
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--github-token', help='Github token')
args = arg_parser.parse_args()
github_token = args.github_token
# gets the workflow status
status, conclusion, step = get_workflow_status(github_token, workflow_id)
# initialize timer
start = time.time()
elapsed: float = 0
# polling the workflow status while is in progress
while status in ['queued', 'in_progress'
] and elapsed < GET_WORKFLOWS_TIMEOUT_THRESHOLD:
logging.info(
f'Workflow {workflow_id} status is {status}, current step: {step}')
time.sleep(60)
status, conclusion, step = get_workflow_status(github_token,
workflow_id)
elapsed = time.time() - start
if elapsed >= GET_WORKFLOWS_TIMEOUT_THRESHOLD:
logging.critical(
f'Timeout reached while waiting for private content build to complete, build url:'
f' {WORKFLOW_HTML_URL}/{workflow_id}')
sys.exit(1)
logging.info(f'Workflow {workflow_id} conclusion is {conclusion}')
if conclusion != 'success':
logging.critical(
f'Private repo build failed, build url: {WORKFLOW_HTML_URL}/{workflow_id}'
)
sys.exit(1)
logging.success('Build private repo finished successfully')
sys.exit(0)
def log_message_if_statement(statement: bool,
error_message: str,
success_message: str = None) -> bool:
"""Log error message if statement is false, Log success otherwise
Args:
statement: The boolean statement to check.
error_message: The error message to log if statement is false
success_message: The success message to log if statement is true
Returns: The statements boolean value.
"""
if not statement:
logging.error(error_message)
elif success_message:
logging.success(success_message)
return statement
def update_content(content_zip_path,
server=None,
username=None,
password=None,
client=None):
"""Update the content on a demisto instance with the content files in a zip file.
Args:
server (str): URL of the demisto server instance.
username (str): Username to login to the demisto instance.
password (str): Password of the associated username to login to the demisto instance.
content_zip_path (str): The path to the zip file containing content files.
client (demisto_client): The configured client to use.
"""
try:
# Configure Demisto Client and make request to upload content zip file
if not client:
client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
file_path = os.path.abspath(content_zip_path)
files = {'file': file_path}
header_params = {'Content-Type': 'multipart/form-data'}
logging.info(
f'Making "POST" request to server - "{server}" to upload the content zip file "{content_zip_path}"'
)
response_data, status_code, _ = client.api_client.call_api(
resource_path='/content/upload',
method='POST',
header_params=header_params,
files=files)
if status_code >= 300 or status_code < 200:
result_object = ast.literal_eval(response_data)
message = result_object['message']
raise Exception(
f"Upload has failed with status code {status_code}\n{message}")
success_msg = f'"{content_zip_path}" successfully uploaded to server "{server}"'
logging.success(success_msg)
except Exception:
logging.exception(
f'Failed to upload {content_zip_path} to server {server}')
def upload_modified_index(public_index_folder_path, extract_destination_path,
public_ci_dummy_index_blob, build_number,
private_packs):
"""Upload updated index zip to cloud storage.
Args:
public_index_folder_path (str): public index folder full path.
extract_destination_path (str): extract folder full path.
public_ci_dummy_index_blob (Blob): google cloud storage object that represents the dummy index.zip blob.
build_number (str): circleCI build number, used as an index revision.
private_packs (list): List of private packs and their price.
"""
with open(os.path.join(public_index_folder_path, "index.json"),
"w+") as index_file:
for private_pack in private_packs:
private_pack['price'] = 0
index = {
'revision': build_number,
'modified': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
'packs': private_packs
}
json.dump(index, index_file, indent=4)
index_zip_name = os.path.basename(public_index_folder_path)
index_zip_path = shutil.make_archive(base_name=public_index_folder_path,
format="zip",
root_dir=extract_destination_path,
base_dir=index_zip_name)
try:
public_ci_dummy_index_blob.reload()
public_ci_dummy_index_blob.cache_control = "no-cache,max-age=0" # disabling caching for index blob
public_ci_dummy_index_blob.upload_from_filename(index_zip_path)
logging.success("Finished uploading index.zip to storage.")
except Exception:
logging.exception(
"Failed in uploading index. Mismatch in index file generation.")
sys.exit(1)
finally:
shutil.rmtree(public_index_folder_path)
def get_all_installed_packs(client: demisto_client):
"""
Args:
client (demisto_client): The client to connect to.
Returns:
list of installed python
"""
try:
logging.info("Attempting to fetch all installed packs.")
response_data, status_code, _ = demisto_client.generic_request_func(
client,
path='/contentpacks/metadata/installed',
method='GET',
accept='application/json',
_request_timeout=None)
if 200 <= status_code < 300:
installed_packs = ast.literal_eval(response_data)
installed_packs_ids = [pack.get('id') for pack in installed_packs]
logging.success('Successfully fetched all installed packs.')
installed_packs_ids_str = ', '.join(installed_packs_ids)
logging.debug(
f'The following packs are currently installed from a previous build run:\n{installed_packs_ids_str}'
)
if 'Base' in installed_packs_ids:
installed_packs_ids.remove('Base')
return installed_packs_ids
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(
f'Failed to fetch installed packs - with status code {status_code}\n{message}'
)
except Exception as e:
logging.exception(
f'The request to fetch installed packs has failed. Additional info: {str(e)}'
)
return None
def xsoar_configure_and_install_flow(options, branch_name: str,
build_number: str):
"""
Args:
options: script arguments.
branch_name(str): name of the current branch.
build_number(str): number of the current build flow
"""
# Get the host by the ami env
server_to_port_mapping, server_version = XSOARBuild.get_servers(
ami_env=options.ami_env)
logging.info('Retrieving the credentials for Cortex XSOAR server')
secret_conf_file = get_json(file_path=options.secret)
username: str = secret_conf_file.get('username')
password: str = secret_conf_file.get('userPassword')
servers = []
# Configure the Servers
for server_url, port in server_to_port_mapping.items():
server = XSOARServer(internal_ip=server_url,
port=port,
user_name=username,
password=password)
logging.info(f'Adding Marketplace configuration to {server_url}')
error_msg: str = 'Failed to set marketplace configuration.'
server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION,
error_msg=error_msg)
XSOARBuild.set_marketplace_url(servers=[server],
branch_name=branch_name,
ci_build_number=build_number)
servers.append(server)
install_packs_from_content_packs_to_install_path(
servers, options.pack_ids_to_install)
logging.success(
f'Finished installing all content packs in {[server.internal_ip for server in servers]}'
)
def main():
parser = argparse.ArgumentParser(
description="Validates landingPage_sections.json file")
parser.add_argument(
'-i',
'--index-path',
help="Path of the unzipped content of the index.zip file",
required=True)
options = parser.parse_args()
landing_page_sections_json: dict = parse_landing_page_sections_to_json()
validate_file_keys(landing_page_sections_json)
bucket_pack_names = {
os.path.basename(pack_name)
for pack_name in glob(f'{options.index_path}/index/*')
}
content_repo_pack_names = {
os.path.basename(pack_name)
for pack_name in glob('Packs/*')
}
valid_packs = bucket_pack_names | content_repo_pack_names
validate_valid_packs_in_sections(landing_page_sections_json, valid_packs)
logging.success('Validation finished successfully')
def install_nightly_packs(client: demisto_client,
host: str,
packs_to_install: List,
request_timeout: int = 999999):
"""
Install content packs on nightly build.
We will catch the exception if pack fails to install and send the request to install packs again without the
corrupted pack.
Args:
client(demisto_client): The configured client to use.
host (str): The server URL.
packs_to_install (list): A list of the packs to install.
request_timeout (int): Timeout settings for the installation request.
Returns:
None: No data returned.
"""
logging.info(f'Installing packs on server {host}')
# make the pack installation request
all_packs_install_successfully = False
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
while not all_packs_install_successfully:
try:
packs_to_install_str = ', '.join([pack['id'] for pack in packs_to_install])
logging.debug(f'Installing the following packs in server {host}:\n{packs_to_install_str}')
response_data, status_code, _ = demisto_client.generic_request_func(client,
path='/contentpacks/marketplace/install',
method='POST',
body=request_data,
accept='application/json',
_request_timeout=request_timeout)
if 200 <= status_code < 300:
packs_data = [{'ID': pack.get('id'), 'CurrentVersion': pack.get('currentVersion')} for pack in
ast.literal_eval(response_data)]
logging.success(f'Packs were successfully installed on server {host}')
logging.debug(f'The following packs were successfully installed on server {host}:\n{packs_data}')
else:
result_object = ast.literal_eval(response_data)
message = result_object.get('message', '')
raise Exception(f'Failed to install packs on server {host}- with status code {status_code}\n{message}\n')
break
except Exception as e:
all_packs_install_successfully = False
malformed_pack_id = find_malformed_pack_id(str(e))
if not malformed_pack_id:
logging.exception('The request to install packs has failed')
raise
pack_ids_to_install = {pack['id'] for pack in packs_to_install}
malformed_pack_id = malformed_pack_id[0]
if malformed_pack_id not in pack_ids_to_install:
logging.exception(
f'The pack {malformed_pack_id} has failed to install even though it was not in the installation list')
raise
logging.warning(f'The request to install packs on server {host} has failed, retrying without {malformed_pack_id}')
# Remove the malformed pack from the pack to install list.
packs_to_install = [pack for pack in packs_to_install if pack['id'] not in malformed_pack_id]
request_data = {
'packs': packs_to_install,
'ignoreWarnings': True
}
def main():
install_logging("TriggerPrivateBuild.log", logger=logging)
# get github token parameter
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--github-token', help='Github token')
args = arg_parser.parse_args()
github_token = args.github_token
# get branch name
branches = tools.run_command("git branch")
branch_name_regex = re.search(r"\* (.*)", branches)
if branch_name_regex:
branch_name = branch_name_regex.group(1)
if branch_has_private_build_infra_change(branch_name):
# get the workflows ids before triggering the build
pre_existing_workflow_ids = get_dispatch_workflows_ids(
github_token, 'master')
# trigger private build
payload = {
'event_type': f'Trigger private build from content/{branch_name}',
'client_payload': {
'commit_sha1': branch_name,
'is_infra_build': 'True'
}
}
res = requests.post(TRIGGER_BUILD_URL,
headers={
'Accept':
'application/vnd.github.everest-preview+json',
'Authorization': f'Bearer {github_token}'
},
data=json.dumps(payload),
verify=False)
if res.status_code != 204:
logging.critical(
f'Failed to trigger private repo build, request to '
f'{TRIGGER_BUILD_URL} failed with error: {str(res.content)}')
sys.exit(1)
workflow_ids_diff = []
for i in range(GET_WORKFLOWS_MAX_RETRIES):
# wait 5 seconds and get the workflow ids again
time.sleep(5)
workflow_ids_after_dispatch = get_dispatch_workflows_ids(
github_token, 'master')
# compare with the first workflows list to get the current id
workflow_ids_diff = [
x for x in workflow_ids_after_dispatch
if x not in pre_existing_workflow_ids
]
if workflow_ids_diff:
break
if len(workflow_ids_diff) == 1:
workflow_id = workflow_ids_diff[0]
logging.success(
f'Private repo build triggered successfully, workflow id: {workflow_id}\n URL:'
f' {WORKFLOW_HTML_URL}/{workflow_id}')
# write the workflow id to text file to use it in get_private_build_status.py
with open(PRIVATE_REPO_WORKFLOW_ID_FILE, "w") as f:
f.write(str(workflow_id))
sys.exit(0)
else:
logging.critical('Could not found the private repo workflow')
sys.exit(1)
else:
logging.info('Build private repo skipped')
def test_instances(secret_conf_path, server, username, password):
integrations = get_integrations(secret_conf_path)
instance_ids = []
failed_integrations = []
integrations_counter = 0
content_installation_client = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
install_new_content(content_installation_client, server)
for integration in integrations:
c = demisto_client.configure(base_url=server,
username=username,
password=password,
verify_ssl=False)
integrations_counter += 1
integration_name = integration.get('name')
integration_instance_name = integration.get('instance_name', '')
integration_params = integration.get('params')
devops_comments = integration.get('devops_comments')
product_description = integration.get('product_description', '')
is_byoi = integration.get('byoi', True)
has_integration = integration.get('has_integration', True)
validate_test = integration.get('validate_test', True)
if has_integration:
try:
instance_id, failure_message = __create_integration_instance(
server,
username,
password,
integration_name,
integration_instance_name,
integration_params,
is_byoi,
validate_test=validate_test)
except Exception:
logging.exception(
f'Failed to configure integration with name {integration_name}'
)
failed_integrations.append(
f"{integration_name} {product_description} - devops comments: {devops_comments}"
)
continue
if failure_message == 'No configuration':
logging.warning(
f"skipping {integration_name} as it exists in content-test-conf conf.json but not in content repo"
)
continue
if not instance_id:
logging.error(
f'Failed to create instance of {integration_name} with message: {failure_message}'
)
failed_integrations.append(
"{} {} - devops comments: {}".format(
integration_name, product_description,
devops_comments))
else:
instance_ids.append(instance_id)
logging.success(
f'Create integration {integration_name} succeed')
__delete_integrations_instances(c, instance_ids)
return failed_integrations, integrations_counter
