def zip_pack(self):
"""Zips pack folder and excludes not wanted directories.
Returns:
bool: whether the operation succeeded.
str: full path to created pack zip.
"""
zip_pack_path = f"{self._pack_path}.zip"
task_status = False
try:
with ZipFile(zip_pack_path, 'w', ZIP_DEFLATED) as pack_zip:
for root, dirs, files in os.walk(self._pack_path,
topdown=True):
dirs[:] = [
d for d in dirs if d not in Pack.EXCLUDE_DIRECTORIES
]
for f in files:
full_file_path = os.path.join(root, f)
relative_file_path = os.path.relpath(
full_file_path, self._pack_path)
pack_zip.write(filename=full_file_path,
arcname=relative_file_path)
task_status = True
print_color(f"Finished zipping {self._pack_name} pack.",
LOG_COLORS.GREEN)
except Exception as e:
print_error(
f"Failed in zipping {self._pack_name} folder.\n Additional info: {e}"
)
finally:
return task_status, zip_pack_path
def slack_notifier(slack_token, secret_conf_path, server, user, password, build_url):
branches = run_command("git branch")
branch_name_reg = re.search("\* (.*)", branches)
branch_name = branch_name_reg.group(1)
if branch_name == 'master':
print_color("Starting Slack notifications about instances", LOG_COLORS.GREEN)
attachments, integrations_counter = get_attachments(secret_conf_path, server, user, password, build_url)
sc = SlackClient(slack_token)
sc.api_call(
"chat.postMessage",
channel="devops-events",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(integrations_counter)
)
sc.api_call(
"chat.postMessage",
channel="content-lab-tests",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(integrations_counter)
)
def update_with_tests_sections(missing_ids, modified_files, test_ids, tests):
test_ids.append(RUN_ALL_TESTS_FORMAT)
# Search for tests section
for file_path in modified_files:
tests_from_file = get_tests(file_path)
for test in tests_from_file:
if test in test_ids or re.match(NO_TESTS_FORMAT, test,
re.IGNORECASE):
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE):
_id = get_script_or_integration_id(file_path)
else:
_id = get_name(file_path)
missing_ids = missing_ids - {_id}
tests.add(test)
else:
message = "The test '{0}' does not exist in the conf.json file, please re-check your code".format(
test)
print_color(message, LOG_COLORS.RED)
global _FAILED
_FAILED = True
return missing_ids
def get_test_list(modified_files, modified_tests_list, all_tests, is_conf_json,
branch_name):
"""Create a test list that should run"""
tests = set([])
if modified_files:
tests = find_tests_for_modified_files(modified_files)
for file_path in modified_tests_list:
test = collect_ids(file_path)
if test not in tests:
tests.add(test)
if is_conf_json:
tests = tests.union(get_test_from_conf(branch_name))
if all_tests:
tests.add("Run all tests")
if not tests and (modified_files or modified_tests_list or all_tests):
print_color(
"There are no tests that check the changes you've done, please make sure you write one",
LOG_COLORS.RED)
sys.exit(1)
return tests
def main():
options = options_handler()
server = options.server
is_ami = options.isAMI
server_version = options.serverVersion
if is_ami: # Run tests in AMI configuration
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ips = [
line.strip('\n').split(":") for line in instance_ips
]
for ami_instance_name, ami_instance_ip in instance_ips:
if ami_instance_name == server_version and ami_instance_name != "Demisto two before GA":
# TODO: remove the and condition once version 4.5 is out
print_color("Starting tests for {}".format(ami_instance_name),
LOG_COLORS.GREEN)
print("Starts tests with server url - https://{}".format(
ami_instance_ip))
server = SERVER_URL.format(ami_instance_ip)
execute_testing(server, ami_instance_ip, server_version)
sleep(8)
else: # Run tests in Server build configuration
with open('public_ip', 'rb') as f:
public_ip = f.read().strip()
execute_testing(server, public_ip, server_version)
def is_correct_content_installed(ips, content_version, api_key):
# type: (AnyStr, List[List], AnyStr) -> bool
""" Checks if specific content version is installed on server list
Args:
ips: list with lists of [instance_name, instance_ip]
content_version: content version that should be installed
api_key: the demisto api key to create an api client with.
Returns:
True: if all tests passed, False if one failure
"""
for ami_instance_name, ami_instance_ip in ips:
host = "https://{}".format(ami_instance_ip)
client = demisto_client.configure(base_url=host,
api_key=api_key,
verify_ssl=False)
try:
resp_json = None
try:
resp = demisto_client.generic_request_func(
self=client,
path='/content/installed/',
method='POST',
accept='application/json',
content_type='application/json')
resp_json = ast.literal_eval(resp[0])
except ApiException as err:
print(err)
if not isinstance(resp_json, dict):
raise ValueError(
'Response from server is not a Dict, got [{}].\n'
'Text: {}'.format(type(resp_json), resp_json))
release = resp_json.get("release")
notes = resp_json.get("releaseNotes")
installed = resp_json.get("installed")
if not (release and content_version in release and notes
and installed):
print_error(
"Failed install content on instance [{}]\nfound content version [{}], expected [{}]"
"".format(ami_instance_name, release, content_version))
return False
else:
print_color(
"Instance [{instance_name}] content verified with version [{content_version}]"
.format(instance_name=ami_instance_name,
content_version=release), LOG_COLORS.GREEN)
except ValueError as exception:
err_msg = "Failed to verify content version on server [{}]\n" \
"Error: [{}]\n".format(ami_instance_name, str(exception))
if resp_json is not None:
err_msg += "Server response: {}".format(resp_json)
print_error(err_msg)
return False
print_color(
"Content was installed successfully on all of the instances! :)",
LOG_COLORS.GREEN)
return True
def get_test_list(files_string, branch_name):
"""Create a test list that should run"""
modified_files, modified_tests_list, all_tests, is_conf_json, infra_tests = get_modified_files(
files_string)
tests = set([])
if modified_files:
tests = find_tests_for_modified_files(modified_files)
for file_path in modified_tests_list:
test = collect_ids(file_path)
if test not in tests:
tests.add(test)
if is_conf_json:
tests = tests.union(get_test_from_conf(branch_name))
if all_tests:
tests.add("Run all tests")
if infra_tests: # Choosing 3 random tests for infrastructure testing
test_ids = get_test_ids(check_nightly_status=True)
for _ in range(3):
tests.add(test_ids[random.randint(0, len(test_ids))])
if not tests and (modified_files or modified_tests_list or all_tests):
print_color(
"There are no tests that check the changes you've done, please make sure you write one",
LOG_COLORS.RED)
sys.exit(1)
return tests
def print_test_summary(succeed_playbooks,
failed_playbooks,
skipped_tests,
skipped_integration,
unmocklable_integrations,
proxy,
is_ami=True):
succeed_count = len(succeed_playbooks)
failed_count = len(failed_playbooks)
skipped_count = len(skipped_tests)
rerecorded_count = len(proxy.rerecorded_tests) if is_ami else 0
empty_mocks_count = len(proxy.empty_files) if is_ami else 0
unmocklable_integrations_count = len(unmocklable_integrations)
print('\nTEST RESULTS:')
print('\t Number of playbooks tested - ' +
str(succeed_count + failed_count))
print_color('\t Number of succeeded tests - ' + str(succeed_count),
LOG_COLORS.GREEN)
if failed_count > 0:
print_error('\t Number of failed tests - ' + str(failed_count) + ':')
for playbook_id in failed_playbooks:
print_error('\t - ' + playbook_id)
if rerecorded_count > 0:
print_warning(
'\t Tests with failed playback and successful re-recording - ' +
str(rerecorded_count) + ':')
for playbook_id in proxy.rerecorded_tests:
print_warning('\t - ' + playbook_id)
if empty_mocks_count > 0:
print('\t Successful tests with empty mock files - ' +
str(empty_mocks_count) + ':')
print(
'\t (either there were no http requests or no traffic is passed through the proxy.\n'
'\t Investigate the playbook and the integrations.\n'
'\t If the integration has no http traffic, add to unmockable_integrations in conf.json)'
)
for playbook_id in proxy.empty_files:
print('\t - ' + playbook_id)
if len(skipped_integration) > 0:
print_warning('\t Number of skipped integration - ' +
str(len(skipped_integration)) + ':')
for playbook_id in skipped_integration:
print_warning('\t - ' + playbook_id)
if skipped_count > 0:
print_warning('\t Number of skipped tests - ' + str(skipped_count) +
':')
for playbook_id in skipped_tests:
print_warning('\t - ' + playbook_id)
if unmocklable_integrations_count > 0:
print_warning('\t Number of unmockable integrations - ' +
str(unmocklable_integrations_count) + ':')
for playbook_id, reason in unmocklable_integrations.items():
print_warning('\t - ' + playbook_id + ' - ' + reason)
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
for regex in CHECKED_TYPES_REGEXES:
splitted_regex = regex.split('.*')
directory = splitted_regex[0]
for root, dirs, files in os.walk(directory):
if root not in DIR_LIST: # Skipping in case we entered a package
continue
print_color('Validating {} directory:'.format(directory), LOG_COLORS.GREEN)
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if file_name.startswith('.'):
continue
print('Validating ' + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
if root in PACKAGE_SUPPORTING_DIRECTORIES:
for inner_dir in dirs:
file_path = glob.glob(os.path.join(root, inner_dir, '*.yml'))[0]
print('Validating ' + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def init_storage_client(service_account=None):
"""Initialize google cloud storage client.
In case of local dev usage the client will be initialized with user default credentials.
Otherwise, client will be initialized from service account json that is stored in CirlceCI.
Args:
service_account (str): full path to service account json.
Return:
storage.Client: initialized google cloud storage client.
"""
if service_account:
storage_client = storage.Client.from_service_account_json(
service_account)
print_color("Created gcp service account", LOG_COLORS.GREEN)
return storage_client
else:
# in case of local dev use, ignored the warning of non use of service account.
warnings.filterwarnings(
"ignore",
message=google.auth._default._CLOUD_SDK_CREDENTIALS_WARNING)
credentials, project = google.auth.default()
storage_client = storage.Client(credentials=credentials,
project=project)
print_color("Created gcp privare account", LOG_COLORS.GREEN)
return storage_client
def main():
"""Execute FilesValidator checks on the modified changes in your branch, or all files in case of master.
This script runs both in a local and a remote environment. In a local environment we don't have any
logger assigned, and then pykwalify raises an error, since it is logging the validation results.
Therefore, if we are in a local env, we set up a logger. Also, we set the logger's level to critical
so the user won't be disturbed by non critical loggings
"""
branches = run_command("git branch")
branch_name_reg = re.search("\* (.*)", branches)
branch_name = branch_name_reg.group(1)
parser = argparse.ArgumentParser(description='Utility CircleCI usage')
parser.add_argument('-c', '--circle', type=str2bool, default=False, help='Is CircleCi or not')
parser.add_argument('-b', '--backwardComp', type=str2bool, default=True, help='To check backward compatibility.')
options = parser.parse_args()
is_circle = options.circle
is_backward_check = options.backwardComp
logging.basicConfig(level=logging.CRITICAL)
print_color("Starting validating files structure", LOG_COLORS.GREEN)
files_validator = FilesValidator(is_circle)
if not files_validator.is_valid_structure(branch_name, is_backward_check=is_backward_check):
sys.exit(1)
print_color("Finished validating files structure", LOG_COLORS.GREEN)
sys.exit(0)
def validate_against_previous_version(self,
branch_sha,
prev_branch_sha=None,
no_error=False):
"""Validate all files that were changed between previous version and branch_sha
Args:
branch_sha (str): Current branch SHA1 to validate
prev_branch_sha (str): Previous branch SHA1 to validate against
no_error (bool): If set to true will restore self._is_valid after run (will not return new errors)
"""
if not prev_branch_sha:
with open('./.circleci/config.yml') as f:
config = yaml.safe_load(f)
prev_branch_sha = config['jobs']['build']['environment'][
'GIT_SHA1']
print_color('Starting validation against {}'.format(prev_branch_sha),
LOG_COLORS.GREEN)
modified_files, _, _ = self.get_modified_and_added_files(
branch_sha, self.is_circle, prev_branch_sha)
prev_self_valid = self._is_valid
self.validate_modified_files(modified_files,
is_backward_check=True,
old_branch=prev_branch_sha)
if no_error:
self._is_valid = prev_self_valid
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
for regex in CHECKED_TYPES_REGEXES:
splitted_regex = regex.split(".*")
directory = splitted_regex[0]
for root, dirs, files in os.walk(directory):
if root not in DIR_LIST: # Skipping in case we entered a package
continue
print_color("Validating {} directory:".format(directory), LOG_COLORS.GREEN)
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if file_name.startswith('.'):
continue
print("Validating " + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
if root in PACKAGE_SUPPORTING_DIRECTORIES:
for inner_dir in dirs:
file_path = glob.glob(os.path.join(root, inner_dir, '*.yml'))[0]
print("Validating " + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def slack_notifier(slack_token, secret_conf_path, server, user, password,
build_url):
branches = run_command("git branch")
branch_name_reg = re.search("\* (.*)", branches)
branch_name = branch_name_reg.group(1)
if branch_name == 'master':
print_color("Starting Slack notifications about instances",
LOG_COLORS.GREEN)
attachments, integrations_counter = get_attachments(
secret_conf_path, server, user, password, build_url)
sc = SlackClient(slack_token)
sc.api_call("chat.postMessage",
channel="devops-events",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(
integrations_counter))
sc.api_call("chat.postMessage",
channel="content-lab-tests",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(
integrations_counter))
def find_duplicates(id_set):
lists_to_return = []
objects_to_check = [
'integrations', 'scripts', 'playbooks', 'TestPlaybooks', 'Classifiers',
'Dashboards', 'Layouts', 'Reports', 'Widgets'
]
for object_type in objects_to_check:
print_color("Checking diff for {}".format(object_type),
LOG_COLORS.GREEN)
objects = id_set.get(object_type)
ids = set(list(specific_item.keys())[0] for specific_item in objects)
dup_list = []
for id_to_check in ids:
if has_duplicate(objects, id_to_check, object_type):
dup_list.append(id_to_check)
lists_to_return.append(dup_list)
print_color("Checking diff for Incident and Idicator Fields",
LOG_COLORS.GREEN)
fields = id_set['IncidentFields'] + id_set['IndicatorFields']
field_ids = set(list(field.keys())[0] for field in fields)
field_list = []
for field_to_check in field_ids:
if has_duplicate(fields, field_to_check,
'Indicator and Incident Fields'):
field_list.append(field_to_check)
lists_to_return.append(field_list)
return lists_to_return
def print_test_summary(tests_data_keeper, is_ami=True):
succeed_playbooks = tests_data_keeper.succeeded_playbooks
failed_playbooks = tests_data_keeper.failed_playbooks
skipped_tests = tests_data_keeper.skipped_tests
unmocklable_integrations = tests_data_keeper.unmockable_integrations
skipped_integration = tests_data_keeper.skipped_integrations
rerecorded_tests = tests_data_keeper.rerecorded_tests
empty_files = tests_data_keeper.empty_files
succeed_count = len(succeed_playbooks)
failed_count = len(failed_playbooks)
skipped_count = len(skipped_tests)
rerecorded_count = len(rerecorded_tests) if is_ami else 0
empty_mocks_count = len(empty_files) if is_ami else 0
unmocklable_integrations_count = len(unmocklable_integrations)
print('\nTEST RESULTS:')
tested_playbooks_message = '\t Number of playbooks tested - ' + str(succeed_count + failed_count)
print(tested_playbooks_message)
succeeded_playbooks_message = '\t Number of succeeded tests - ' + str(succeed_count)
print_color(succeeded_playbooks_message, LOG_COLORS.GREEN)
if failed_count > 0:
failed_tests_message = '\t Number of failed tests - ' + str(failed_count) + ':'
print_error(failed_tests_message)
for playbook_id in failed_playbooks:
print_error('\t - ' + playbook_id)
if rerecorded_count > 0:
recording_warning = '\t Tests with failed playback and successful re-recording - ' + str(rerecorded_count) + ':'
print_warning(recording_warning)
for playbook_id in rerecorded_tests:
print_warning('\t - ' + playbook_id)
if empty_mocks_count > 0:
empty_mock_successes_msg = '\t Successful tests with empty mock files - ' + str(empty_mocks_count) + ':'
print(empty_mock_successes_msg)
proxy_explanation = '\t (either there were no http requests or no traffic is passed through the proxy.\n'\
'\t Investigate the playbook and the integrations.\n'\
'\t If the integration has no http traffic, add to unmockable_integrations in conf.json)'
print(proxy_explanation)
for playbook_id in empty_files:
print('\t - ' + playbook_id)
if len(skipped_integration) > 0:
skipped_integrations_warning = '\t Number of skipped integration - ' + str(len(skipped_integration)) + ':'
print_warning(skipped_integrations_warning)
for playbook_id in skipped_integration:
print_warning('\t - ' + playbook_id)
if skipped_count > 0:
skipped_tests_warning = '\t Number of skipped tests - ' + str(skipped_count) + ':'
print_warning(skipped_tests_warning)
for playbook_id in skipped_tests:
print_warning('\t - ' + playbook_id)
if unmocklable_integrations_count > 0:
unmockable_warning = '\t Number of unmockable integrations - ' + str(unmocklable_integrations_count) + ':'
print_warning(unmockable_warning)
for playbook_id, reason in unmocklable_integrations.items():
print_warning('\t - ' + playbook_id + ' - ' + reason)
def run_test_logic(c,
failed_playbooks,
integrations,
playbook_id,
succeed_playbooks,
test_message,
test_options,
slack,
circle_ci,
build_number,
server_url,
build_name,
is_mock_run=False):
status, inc_id = test_integration(c, integrations, playbook_id,
test_options, is_mock_run)
if status == PB_Status.COMPLETED:
print_color('PASS: {} succeed'.format(test_message), LOG_COLORS.GREEN)
succeed_playbooks.append(playbook_id)
elif status == PB_Status.NOT_SUPPORTED_VERSION:
print('PASS: {} skipped - not supported version'.format(test_message))
succeed_playbooks.append(playbook_id)
else:
print_error('Failed: {} failed'.format(test_message))
playbook_id_with_mock = playbook_id
if not is_mock_run:
playbook_id_with_mock += " (Mock Disabled)"
failed_playbooks.append(playbook_id_with_mock)
notify_failed_test(slack, circle_ci, playbook_id, build_number, inc_id,
server_url, build_name)
succeed = status == PB_Status.COMPLETED or status == PB_Status.NOT_SUPPORTED_VERSION
return succeed
def slack_notifier(build_url, slack_token, env_results_file_name, container):
branches = run_command("git branch")
branch_name_reg = re.search(r'\* (.*)', branches)
branch_name = branch_name_reg.group(1)
if branch_name == 'master':
print("Extracting build status")
# container 1: unit tests
if int(container):
print_color("Starting Slack notifications about nightly build - unit tests", LOG_COLORS.GREEN)
content_team_attachments = get_attachments_for_unit_test(build_url)
# container 0: test playbooks
else:
print_color("Starting Slack notifications about nightly build - tests playbook", LOG_COLORS.GREEN)
content_team_attachments, _ = get_attachments_for_test_playbooks(build_url, env_results_file_name)
print("Sending Slack messages to #content-team")
slack_client = SlackClient(slack_token)
slack_client.api_call(
"chat.postMessage",
channel="dmst-content-team",
username="******",
as_user="******",
attachments=content_team_attachments
)
def main():
options = options_handler()
server = options.server
is_ami = options.isAMI
server_version = options.serverVersion
server_numeric_version = '0.0.0'
if is_ami: # Run tests in AMI configuration
with open('./Tests/images_data.txt', 'r') as image_data_file:
image_data = [
line for line in image_data_file
if line.startswith(server_version)
]
if len(image_data) != 1:
print('Did not get one image data for server version, got {}'.
format(image_data))
else:
server_numeric_version = re.findall(
'Demisto-Circle-CI-Content-[\w-]+-([\d.]+)-[\d]{5}',
image_data[0])
if server_numeric_version:
server_numeric_version = server_numeric_version[0]
else:
server_numeric_version = '99.99.98' # latest
print('Server image info: {}'.format(image_data[0]))
print('Server version: {}'.format(server_numeric_version))
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ips = [
line.strip('\n').split(":") for line in instance_ips
]
for ami_instance_name, ami_instance_ip in instance_ips:
if ami_instance_name == server_version:
print_color("Starting tests for {}".format(ami_instance_name),
LOG_COLORS.GREEN)
print("Starts tests with server url - https://{}".format(
ami_instance_ip))
server = SERVER_URL.format(ami_instance_ip)
execute_testing(server, ami_instance_ip, server_version,
server_numeric_version)
sleep(8)
else: # Run tests in Server build configuration
server_numeric_version = '99.99.98' # assume latest
print("Using server version: {} (assuming latest for non-ami)".format(
server_numeric_version))
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ip = [
line.strip('\n').split(":")[1] for line in instance_ips
][0]
execute_testing(SERVER_URL.format(instance_ip),
instance_ip,
server_version,
server_numeric_version,
is_ami=False)
def main():
options = options_handler()
if options.nightly:
slack_notifier(options.url, options.slack,
options.env_results_file_name)
else:
print_color(
"Not nightly build, stopping Slack Notifications about Content build",
LOG_COLORS.RED)
def handle_run_res(res: Tuple[subprocess.CompletedProcess, str], fail_pkgs: list, good_pkgs: list):
if res[0].returncode != 0:
fail_pkgs.append(res[1])
print_color("============= {} =============".format(res[1]), LOG_COLORS.RED)
else:
good_pkgs.append(res[1])
print("============= {} =============".format(res[1]))
print(res[0].stdout)
print(res[0].stderr)
def main():
if len(sys.argv) == 2 and (sys.argv[1] == '-h' or sys.argv[1] == '--help'):
print(
"Run pkg_dev_test_tasks.py in parallel. Accepts same parameters as pkg_dev_test_tasks.py.\n"
"Additionally you can specify the following environment variables:\n"
"DIFF_COMPARE: specify how to do a git compare. Leave empty to run on all.\n"
"MAX_WORKERS: max amount of workers to use for running ")
sys.exit(1)
max_workers = int(os.getenv("MAX_WORKERS", "10"))
find_out = subprocess.check_output([
"find", "Integrations", "Scripts", "Beta_Integrations", "-maxdepth",
"1", "-mindepth", "1", "-type", "d", "-print"
],
text=True)
pkg_dirs = find_out.splitlines()
pkgs_to_run = []
for dir in pkg_dirs:
if should_run_pkg(dir):
pkgs_to_run.append(dir)
print(
"Starting parallel run for [{}] packages with [{}] max workers".format(
len(pkgs_to_run), max_workers))
params = sys.argv[1::]
fail_pkgs = []
good_pkgs = []
if len(pkgs_to_run) > 1: # setup pipenv before hand to avoid conflics
get_dev_requirements(2.7)
get_dev_requirements(3.7)
# run CommonServer non parallel to avoid conflicts
# when we modify the file for mypy includes
if 'Scripts/CommonServerPython' in pkgs_to_run:
pkgs_to_run.remove('Scripts/CommonServerPython')
res = run_dev_task('Scripts/CommonServerPython', params)
handle_run_res(res, fail_pkgs, good_pkgs)
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers) as executor:
futures_submit = [
executor.submit(run_dev_task, dir, params) for dir in pkgs_to_run
]
for future in concurrent.futures.as_completed(futures_submit):
res = future.result()
handle_run_res(res, fail_pkgs, good_pkgs)
if fail_pkgs:
create_failed_unittests_file(fail_pkgs)
print_color("\n******* FAIL PKGS: *******", LOG_COLORS.RED)
print_color("\n\t{}\n".format("\n\t".join(fail_pkgs)), LOG_COLORS.RED)
if good_pkgs:
print_color("\n******* SUCCESS PKGS: *******", LOG_COLORS.GREEN)
print_color("\n\t{}\n".format("\n\t".join(good_pkgs)),
LOG_COLORS.GREEN)
if not good_pkgs and not fail_pkgs:
print_color("\n******* No changed packages found *******\n",
LOG_COLORS.YELLOW)
if fail_pkgs:
sys.exit(1)
def upload_to_storage(self, zip_pack_path, latest_version, storage_bucket,
override_pack):
""" Manages the upload of pack zip artifact to correct path in cloud storage.
The zip pack will be uploaded to following path: /content/packs/pack_name/pack_latest_version.
In case that zip pack artifact already exist at constructed path, the upload will be skipped.
If flag override_pack is set to True, pack will forced for upload.
Args:
zip_pack_path (str): full path to pack zip artifact.
latest_version (str): pack latest version.
storage_bucket (google.cloud.storage.bucket.Bucket): google cloud storage bucket.
override_pack (bool): whether to override existing pack.
Returns:
bool: whether the operation succeeded.
bool: True in case of pack existence at targeted path and upload was skipped, otherwise returned False.
"""
task_status = True
try:
version_pack_path = os.path.join(STORAGE_BASE_PATH,
self._pack_name, latest_version)
existing_files = [
f.name
for f in storage_bucket.list_blobs(prefix=version_pack_path)
]
if existing_files and not override_pack:
print_warning(
f"The following packs already exist at storage: {', '.join(existing_files)}"
)
print_warning(
f"Skipping step of uploading {self._pack_name}.zip to storage."
)
return task_status, True
pack_full_path = f"{version_pack_path}/{self._pack_name}.zip"
blob = storage_bucket.blob(pack_full_path)
with open(zip_pack_path, "rb") as pack_zip:
blob.upload_from_file(pack_zip)
self.relative_storage_path = blob.name
print_color(
f"Uploaded {self._pack_name} pack to {pack_full_path} path.",
LOG_COLORS.GREEN)
return task_status, False
except Exception as e:
task_status = False
print_error(
f"Failed in uploading {self._pack_name} pack to gcs.\nAdditional info: {e}"
)
return task_status, True
def main():
options = parse_script_arguments()
is_circle = options.circle
branch_name = get_branch_name()
is_forked = re.match(EXTERNAL_PR_REGEX, branch_name) is not None
if not is_forked:
secrets_found = get_secrets(branch_name, is_circle)
if secrets_found:
sys.exit(1)
else:
print_color('Finished validating secrets, no secrets were found.', LOG_COLORS.GREEN)
sys.exit(0)
def is_correct_content_installed(username, password, ips, content_version):
# type: (AnyStr, AnyStr, List[List], AnyStr) -> bool
""" Checks if specific content version is installed on server list
Args:
username: for server connection
password: for server connection
ips: list with lists of [instance_name, instance_ip]
content_version: content version that should be installed
Returns:
True: if all tests passed, False if one failure
"""
method = "post"
suffix = "/content/installed/"
for ami_instance_name, ami_instance_ip in ips:
d = demisto.DemistoClient(None, "https://{}".format(ami_instance_ip),
username, password)
d.Login()
resp = d.req(method, suffix, None)
resp_json = None
try:
resp_json = resp.json()
if not isinstance(resp_json, dict):
raise ValueError(
'Response from server is not a Dict, got [{}].\n'
'Text: {}'.format(type(resp_json), resp.text))
release = resp_json.get("release")
notes = resp_json.get("releaseNotes")
installed = resp_json.get("installed")
if not (release and content_version in release and notes
and installed):
print_error(
"Failed install content on instance [{}]\nfound content version [{}], expected [{}]"
"".format(ami_instance_name, release, content_version))
return False
else:
print_color(
"Instance [{instance_name}] content verified with version [{content_version}]"
.format(instance_name=ami_instance_name,
content_version=release), LOG_COLORS.GREEN)
except ValueError as exception:
err_msg = "Failed to verify content version on server [{}]\n" \
"Error: [{}]\n".format(ami_instance_name, str(exception))
if resp_json is not None:
err_msg += "Server response: {}".format(resp_json)
print_error(err_msg)
return False
print_color(
"Content was installed successfully on all of the instances! :)",
LOG_COLORS.GREEN)
return True
def slack_notifier(slack_token, secret_conf_path, server, user, password, build_url):
print_color("Starting Slack notifications about instances", LOG_COLORS.GREEN)
attachments, integrations_counter = get_attachments(secret_conf_path, server, user, password, build_url)
sc = SlackClient(slack_token)
sc.api_call(
"chat.postMessage",
channel="dmst-content-lab",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(integrations_counter)
)
def download_and_extract_index(storage_bucket, extract_destination_path):
"""Downloads and extracts index zip from cloud storage.
Args:
storage_bucket (google.cloud.storage.bucket.Bucket): google storage bucket where index.zip is stored.
extract_destination_path (str): the full path of extract folder.
Returns:
str: extracted index folder full path.
Blob: google cloud storage object that represents index.zip blob.
"""
index_storage_path = os.path.join(STORAGE_BASE_PATH,
f"{Pack.INDEX_NAME}.zip")
download_index_path = os.path.join(extract_destination_path,
f"{Pack.INDEX_NAME}.zip")
index_blob = storage_bucket.blob(index_storage_path)
index_folder_path = os.path.join(extract_destination_path, Pack.INDEX_NAME)
if not index_blob.exists():
os.mkdir(index_folder_path)
return index_folder_path, index_blob
index_blob.cache_control = "no-cache" # index zip should never be cached in the memory, should be updated version
index_blob.reload()
index_blob.download_to_filename(download_index_path)
if os.path.exists(download_index_path):
with ZipFile(download_index_path, 'r') as index_zip:
index_zip.extractall(extract_destination_path)
if not os.path.exists(index_folder_path):
print_error(
f"Failed creating {Pack.INDEX_NAME} folder with extracted data."
)
sys.exit(1)
os.remove(download_index_path)
print_color(
f"Finished downloading and extracting {Pack.INDEX_NAME} file to {extract_destination_path}",
LOG_COLORS.GREEN)
return index_folder_path, index_blob
else:
print_error(
f"Failed to download {Pack.INDEX_NAME}.zip file from cloud storage."
)
sys.exit(1)
def mock_run(c, proxy, failed_playbooks, integrations, playbook_id,
succeed_playbooks, test_message, test_options, slack, circle_ci,
build_number, server_url, build_name, start_message):
rerecord = False
if proxy.has_mock_file(playbook_id):
print('{} (Mock: Playback)'.format(start_message))
proxy.start(playbook_id)
# run test
status, inc_id = test_integration(c,
integrations,
playbook_id,
test_options,
is_mock_run=True)
# use results
proxy.stop()
if status == PB_Status.COMPLETED:
print_color('PASS: {} succeed'.format(test_message),
LOG_COLORS.GREEN)
succeed_playbooks.append(playbook_id)
print('------ Test {} end ------\n'.format(test_message))
return
elif status == PB_Status.NOT_SUPPORTED_VERSION:
print('PASS: {} skipped - not supported version'.format(
test_message))
succeed_playbooks.append(playbook_id)
print('------ Test {} end ------\n'.format(test_message))
return
else:
print(
"Test failed with mock, recording new mock file. (Mock: Recording)"
)
rerecord = True
else:
print(start_message + ' (Mock: Recording)')
# Mock recording - no mock file or playback failure.
succeed = run_and_record(c, proxy, failed_playbooks, integrations,
playbook_id, succeed_playbooks, test_message,
test_options, slack, circle_ci, build_number,
server_url, build_name)
if rerecord and succeed:
proxy.rerecorded_tests.append(playbook_id)
print('------ Test {} end ------\n'.format(test_message))
def main():
"""Execute FilesValidator checks on the modified changes in your branch, or all files in case of master.
This script runs both in a local and a remote environment. In a local environment we don't have any
logger assigned, and then pykwalify raises an error, since it is logging the validation results.
Therefore, if we are in a local env, we set up a logger. Also, we set the logger's level to critical
so the user won't be disturbed by non critical loggings
"""
branches = run_command('git branch')
branch_name_reg = re.search(r'\* (.*)', branches)
branch_name = branch_name_reg.group(1)
parser = argparse.ArgumentParser(description='Utility CircleCI usage')
parser.add_argument('-c', '--circle', type=str2bool, default=False, help='Is CircleCi or not')
parser.add_argument('-b', '--backwardComp', type=str2bool, default=True, help='To check backward compatibility.')
parser.add_argument('-t', '--test-filter', type=str2bool, default=False, help='Check that tests are valid.')
parser.add_argument('-p', '--prev-ver', help='Previous branch or SHA1 commit to run checks against.')
options = parser.parse_args()
is_circle = options.circle
is_backward_check = options.backwardComp
logging.basicConfig(level=logging.CRITICAL)
print_color('Starting validating files structure', LOG_COLORS.GREEN)
files_validator = FilesValidator(is_circle, print_ignored_files=True)
if not files_validator.is_valid_structure(branch_name, is_backward_check=is_backward_check,
prev_ver=options.prev_ver):
sys.exit(1)
if options.test_filter:
try:
print_warning('Updating idset. Be patient if this is the first time...')
subprocess.check_output(['./Tests/scripts/update_id_set.py'])
print_warning('Checking that we have tests for all content...')
try:
tests_out = subprocess.check_output(['./Tests/scripts/configure_tests.py', '-s', 'true'],
stderr=subprocess.STDOUT)
print(tests_out)
except Exception:
print_warning('Recreating idset to be sure that configure tests failure is accurate.'
' Be patient this can take 15-20 seconds ...')
subprocess.check_output(['./Tests/scripts/update_id_set.py', '-r'])
print_warning('Checking that we have tests for all content again...')
subprocess.check_call(['./Tests/scripts/configure_tests.py', '-s', 'true'])
except Exception as ex:
print_error('Failed validating tests: {}'.format(ex))
sys.exit(1)
print_color('Finished validating files structure', LOG_COLORS.GREEN)
sys.exit(0)
def format_metadata(self, pack_content_items, integration_images):
"""Re-formats metadata according to marketplace metadata format defined in issue #19786 and writes back
the result.
Args:
pack_content_items (dict): content items that are located inside specific pack. Possible keys of the dict:
Classifiers, Dashboards, IncidentFields, IncidentTypes, IndicatorFields, Integrations, Layouts, Playbooks,
Reports, Scripts and Widgets. Each key is mapped to list of items with name and description. Several items
have no description.
integration_images (list): list of uploaded integration images with integration display name and image gcs
public url.
Returns:
bool: True is returned in case metadata file was parsed successfully, otherwise False.
"""
user_metadata_path = os.path.join(
self._pack_path,
Pack.USER_METADATA) # user metadata path before parsing
metadata_path = os.path.join(
self._pack_path,
Pack.METADATA) # deployed metadata path after parsing
if not os.path.exists(user_metadata_path):
print_error(
f"{self._pack_name} pack is missing {Pack.USER_METADATA} file."
)
return False
with open(user_metadata_path, "r") as user_metadata_file:
user_metadata = json.load(
user_metadata_file) # loading user metadata
formatted_metadata = Pack._parse_pack_metadata(
user_metadata=user_metadata,
pack_content_items=pack_content_items,
pack_id=self._pack_name,
integration_images=integration_images)
with open(metadata_path, "w") as metadata_file:
json.dump(formatted_metadata, metadata_file,
indent=4) # writing back parsed metadata
print_color(
f"Finished formatting {self._pack_name} packs's {Pack.METADATA} {metadata_path} file.",
LOG_COLORS.GREEN)
return True
def update_content_on_demisto_instance(client, username, password, server):
'''Try to update the content
Args:
client (demisto_client): The configured client to use.
username (str): The username to pass to Tests/update_content_data.py
password (str): The password to pass to Tests/update_content_data.py
server (str): The server url to pass to Tests/update_content_data.py
'''
content_zip_path = 'artifacts/all_content.zip'
cmd_str = 'python Tests/update_content_data.py -u {} -p {} -s {} --content_zip {}'.format(
username, password, server, content_zip_path)
run_command(cmd_str, is_silenced=False)
# Check if content update has finished installing
sleep_interval = 20
updating_content = is_content_update_in_progress(client)
while updating_content.lower() == 'true':
sleep(sleep_interval)
updating_content = is_content_update_in_progress(client)
if updating_content.lower() == 'request unsuccessful':
# since the request to check if content update installation finished didn't work, can't use that mechanism
# to check and just try sleeping for 30 seconds instead to allow for content update installation to complete
sleep(30)
else:
# check that the content installation updated
# verify the asset id matches the circleci build number / asset_id in the content-descriptor.json
release, asset_id = get_content_version_details(client)
with open('content-descriptor.json', 'r') as cd_file:
cd_json = json.loads(cd_file.read())
cd_release = cd_json.get('release')
cd_asset_id = cd_json.get('assetId')
if release == cd_release and asset_id == cd_asset_id:
print_color('Content Update Successfully Installed!',
color=LOG_COLORS.GREEN)
else:
err_details = 'Attempted to install content with release "{}" and assetId '.format(
cd_release)
err_details += '"{}" but release "{}" and assetId "{}" were '.format(
cd_asset_id, release, asset_id)
err_details += 'retrieved from the instance post installation.'
print_error(
'Content Update was Unsuccessful:\n{}'.format(err_details))
sys.exit(1)
def verify_base_branch(pr_num):
"""Checks if the base branch is master or not
Args:
pr_num (string): The string representation of the pr number
Returns:
True if valid, False otherwise. And attaches the appropriate message to the user.
"""
print_color('Fetching the base branch of pull request #{}.'.format(pr_num),
LOG_COLORS.NATIVE)
base_branch = get_base_branch(pr_num)
if base_branch == 'master':
return 'Cannot merge a contribution directly to master, the pull request reviewer will handle that soon.', False
else:
return 'Verified pull request #{} base branch successfully.'.format(
pr_num), True
def slack_notifier(build_url, slack_token):
branches = run_command("git branch")
branch_name_reg = re.search("\* (.*)", branches)
branch_name = branch_name_reg.group(1)
if branch_name == 'master':
print_color("Starting Slack notifications about nightly build", LOG_COLORS.GREEN)
print("Extracting build status")
content_team_attachments, content_attachments = get_attachments(build_url)
print("Sending Slack messages to #content and #content-team")
sc = SlackClient(slack_token)
sc.api_call(
"chat.postMessage",
channel="content-team",
username="******",
as_user="******",
attachments=content_team_attachments
)
def find_tests_for_modified_files(modified_files):
script_names = set([])
playbook_names = set([])
integration_ids = set([])
tests_set, catched_scripts, catched_playbooks = collect_changed_ids(integration_ids, playbook_names,
script_names, modified_files)
test_ids, missing_ids, caught_missing_test = collect_tests(script_names, playbook_names, integration_ids,
catched_scripts, catched_playbooks, tests_set)
missing_ids = update_with_tests_sections(missing_ids, modified_files, test_ids, tests_set)
if len(missing_ids) > 0:
test_string = '\n'.join(missing_ids)
message = "You've failed to provide tests for:\n{0}".format(test_string)
print_color(message, LOG_COLORS.RED)
if caught_missing_test or len(missing_ids) > 0:
sys.exit(1)
return tests_set
def update_with_tests_sections(missing_ids, modified_files, test_ids, tests):
test_ids.append(RUN_ALL_TESTS_FORMAT)
# Search for tests section
for file_path in modified_files:
tests_from_file = get_tests(file_path)
for test in tests_from_file:
if test in test_ids or re.match(NO_TESTS_FORMAT, test, re.IGNORECASE):
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE):
_id = get_script_or_integration_id(file_path)
else:
_id = get_name(file_path)
missing_ids = missing_ids - {_id}
tests.add(test)
else:
message = "The test '{0}' does not exist in the conf.json file, please re-check your code".format(test)
print_color(message, LOG_COLORS.RED)
sys.exit(1)
return missing_ids
branches = run_command("git branch")
branch_name_reg = re.search("\* (.*)", branches)
branch_name = branch_name_reg.group(1)
if branch_name == 'master':
print_color("Starting Slack notifications about nightly build", LOG_COLORS.GREEN)
print("Extracting build status")
content_team_attachments, content_attachments = get_attachments(build_url)
print("Sending Slack messages to #content and #content-team")
sc = SlackClient(slack_token)
sc.api_call(
"chat.postMessage",
channel="content-team",
username="******",
as_user="******",
attachments=content_team_attachments
)
if __name__ == "__main__":
options = options_handler()
if options.nightly:
slack_notifier(options.url, options.slack)
else:
print_color("Not nightly build, stopping Slack Notifications about Content build", LOG_COLORS.RED)
os.remove('./Tests/failed_tests.txt')
os.remove('./Tests/skipped_tests.txt')
os.remove('./Tests/skipped_integrations.txt')
def re_create_id_set():
start_time = time.time()
scripts_list = []
playbooks_list = []
integration_list = []
testplaybooks_list = []
pool = Pool(processes=cpu_count() * 2)
print_color("Starting the creation of the id_set", LOG_COLORS.GREEN)
print_color("Starting iterating over Integrations", LOG_COLORS.GREEN)
integration_files = glob.glob(os.path.join('Integrations', '*'))
integration_files.extend(glob.glob(os.path.join('Beta_Integrations', '*')))
for arr in pool.map(process_integration, integration_files):
integration_list.extend(arr)
print_color("Starting iterating over Playbooks", LOG_COLORS.GREEN)
for arr in pool.map(process_playbook, glob.glob(os.path.join('Playbooks', '*'))):
playbooks_list.extend(arr)
print_color("Starting iterating over Scripts", LOG_COLORS.GREEN)
for arr in pool.map(process_script, glob.glob(os.path.join('Scripts', '*'))):
scripts_list.extend(arr)
print_color("Starting iterating over TestPlaybooks", LOG_COLORS.GREEN)
for pair in pool.map(process_testplaybook_path, glob.glob(os.path.join('TestPlaybooks', '*'))):
if pair[0]:
testplaybooks_list.append(pair[0])
if pair[1]:
scripts_list.append(pair[1])
new_ids_dict = OrderedDict()
# we sort each time the whole set in case someone manually changed something
# it shouldn't take too much time
new_ids_dict['scripts'] = sort(scripts_list)
new_ids_dict['playbooks'] = sort(playbooks_list)
new_ids_dict['integrations'] = sort(integration_list)
new_ids_dict['TestPlaybooks'] = sort(testplaybooks_list)
with open('./Tests/id_set.json', 'w') as id_set_file:
json.dump(new_ids_dict, id_set_file, indent=4)
exec_time = time.time() - start_time
print_color("Finished the creation of the id_set. Total time: {} seconds".format(exec_time), LOG_COLORS.GREEN)
commit_string = run_command("git log -n 2 --pretty='%H'")
commit_string = commit_string.replace("'", "")
last_commit, second_last_commit = commit_string.split()
files_string = run_command("git diff --name-status {}...{}".format(second_last_commit, last_commit))
tests = get_test_list(files_string, branch_name)
tests_string = '\n'.join(tests)
if tests_string:
print('Collected the following tests:\n{0}\n'.format(tests_string))
else:
print('No filter configured, running all tests')
print("Creating filter_file.txt")
with open("./Tests/filter_file.txt", "w") as filter_file:
filter_file.write(tests_string)
if __name__ == "__main__":
print_color("Starting creation of test filter file", LOG_COLORS.GREEN)
parser = argparse.ArgumentParser(description='Utility CircleCI usage')
parser.add_argument('-n', '--nightly', type=str2bool, help='Is nightly or not')
options = parser.parse_args()
# Create test file based only on committed files
create_test_file(options.nightly)
print_color("Finished creation of the test filter file", LOG_COLORS.GREEN)
sys.exit(0)
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(integrations_counter)
)
sc.api_call(
"chat.postMessage",
channel="content-lab-tests",
username="******",
as_user="******",
attachments=attachments,
text="You have {0} instances configurations".format(integrations_counter)
)
if __name__ == "__main__":
options = options_handler()
if options.nightly:
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ips = [line.strip('\n').split(":") for line in instance_ips]
for ami_instance_name, ami_instance_ip in instance_ips:
if ami_instance_name == "Demisto GA":
server = SERVER_URL.format(ami_instance_ip)
slack_notifier(options.slack, options.secret, server, options.user, options.password, options.buildUrl)
else:
print_color("Not nightly build, stopping Slack Notifications about instances", LOG_COLORS.RED)
