def main():
instance_ips = []
instance_ids = []
id_to_ip = {}
with open('./env_results.json', 'r') as json_file:
env_results = json.load(json_file)
for env in env_results:
id_to_ip.update({env["InstanceID"]: env["InstanceDNS"]})
instance_ips.append(env["Role"] + ":" + env["InstanceDNS"])
instance_ids.append(env["Role"] + ":" + env["InstanceID"])
with open('./Tests/images_data.txt', 'a') as instance_file:
instance_file.write('{} Image info is: {} {} {}\n'.format(
env["Role"], env["AmiId"], env["AmiName"], env["AmiCreation"]))
with open('./Tests/instance_ids.txt', 'w') as instance_file:
instance_file.write('\n'.join(instance_ids))
print("Waiting 90 Seconds for SSH to start\n")
sleep(90)
threads_list = []
for instance_ip in id_to_ip.values():
t = Thread(target=run_command,
args=("./Tests/scripts/copy_content_data.sh {}".format(
instance_ip), ),
kwargs={'is_silenced': False})
threads_list.append(t)
run_threads_list(threads_list)
with open('./Tests/instance_ips.txt', 'w') as instance_file:
instance_file.write('\n'.join(instance_ips))
def main():
circle_aritfact = sys.argv[1]
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ips = [
line.strip('\n').split(":") for line in instance_ips
if line.strip() != ''
]
with open('./Tests/instance_ids.txt', 'r') as instance_file:
instance_ids = instance_file.readlines()
instance_ids = [
line.strip('\n').split(":") for line in instance_ids
if line.strip() != ''
]
threads_list = []
for ami_instance_name, ami_instance_ip in instance_ips:
for ami_instance_name_second, ami_instance_id in instance_ids:
if ami_instance_name == ami_instance_name_second:
t = Thread(
target=run_command,
args=
("./Tests/scripts/destroy_instances.sh {} {} {} ./Tests/is_build_failed_{}.txt"
.format(circle_aritfact, ami_instance_id, ami_instance_ip,
ami_instance_name.replace(' ', '')), ),
kwargs={'is_silenced': False})
threads_list.append(t)
run_threads_list(threads_list)
def main():
instance_ips = []
with open('./Tests/instance_ids.txt', 'r') as instance_file:
ami_instances = instance_file.readlines()
ami_instances = [line.strip('\n').split(":") for line in ami_instances if line.strip('\n').split(":") != ['']]
id_to_ip = {}
for ami_instance_name, ami_instance_id in ami_instances:
print "Validating ami instance: {}".format(ami_instance_name)
run_command("./Tests/scripts/get_instance_ip.sh {}".format(ami_instance_id))
# get_instance_ip.sh script is writing the ip to instance_ips.txt because we couldn't get the ip
# from the output of the aws script
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ip = instance_file.read()
instance_ip = instance_ip.strip()
print("The IP of the instance is {}\n".format(instance_ip))
id_to_ip[ami_instance_id] = instance_ip
print("Waiting 90 Seconds for SSH to start\n")
sleep(90)
threads_list = []
for ami_instance_name, ami_instance_id in ami_instances:
t = Thread(target=run_command,
args=("./Tests/scripts/copy_content_data.sh {}".format(id_to_ip[ami_instance_id]), ),
kwargs={'is_silenced': False})
threads_list.append(t)
# copy_content_data.sh also starts the server
instance_ips.append("{}:{}".format(ami_instance_name, id_to_ip[ami_instance_id]))
run_threads_list(threads_list)
with open('./Tests/instance_ips.txt', 'w') as instance_file:
instance_file.write('\n'.join(instance_ips))
def main():
circle_aritfact = sys.argv[1]
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ips = instance_file.readlines()
instance_ips = [line.strip('\n').split(":") for line in instance_ips if line.strip() != '']
with open('./Tests/instance_ids.txt', 'r') as instance_file:
instance_ids = instance_file.readlines()
instance_ids = [line.strip('\n').split(":") for line in instance_ids if line.strip() != '']
threads_list = []
for ami_instance_name, ami_instance_ip in instance_ips:
for ami_instance_name_second, ami_instance_id in instance_ids:
if ami_instance_name == ami_instance_name_second:
t = Thread(target=run_command,
args=("./Tests/scripts/destroy_instances.sh {} {} {} ./Tests/is_build_failed_{}.txt".format(
circle_aritfact,
ami_instance_id,
ami_instance_ip,
ami_instance_name.replace(' ', '')),
), kwargs={'is_silenced': False})
threads_list.append(t)
run_threads_list(threads_list)
def main():
instance_ips = []
with open('./Tests/instance_ids.txt', 'r') as instance_file:
ami_instances = instance_file.readlines()
ami_instances = [
line.strip('\n').split(":") for line in ami_instances
if line.strip('\n').split(":") != ['']
]
id_to_ip = {}
for ami_instance_name, ami_instance_id in ami_instances:
print "Validating ami instance: {}".format(ami_instance_name)
run_command(
"./Tests/scripts/get_instance_ip.sh {}".format(ami_instance_id))
# get_instance_ip.sh script is writing the ip to instance_ips.txt because we couldn't get the ip
# from the output of the aws script
with open('./Tests/instance_ips.txt', 'r') as instance_file:
instance_ip = instance_file.read()
instance_ip = instance_ip.strip()
print("The IP of the instance is {}\n".format(instance_ip))
id_to_ip[ami_instance_id] = instance_ip
print("Waiting 90 Seconds for SSH to start\n")
sleep(90)
threads_list = []
for ami_instance_name, ami_instance_id in ami_instances:
t = Thread(target=run_command,
args=("./Tests/scripts/copy_content_data.sh {}".format(
id_to_ip[ami_instance_id]), ),
kwargs={'is_silenced': False})
threads_list.append(t)
# copy_content_data.sh also starts the server
instance_ips.append("{}:{}".format(ami_instance_name,
id_to_ip[ami_instance_id]))
run_threads_list(threads_list)
with open('./Tests/instance_ips.txt', 'w') as instance_file:
instance_file.write('\n'.join(instance_ips))
def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
servers = determine_servers_urls(ami_env)
conf_path = options.conf
secret_conf_path = options.secret
prints_manager = ParallelPrintsManager(1)
server_numeric_version = get_server_numeric_version(ami_env)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
clients = []
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
clients.append(client)
testing_client = clients[0]
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(
is_nightly=False)
tests_for_iteration = tests
if run_all_tests:
# Use all tests for testing, leave 'tests_for_iteration' as is
pass
elif filter_configured and filtered_tests:
tests_for_iteration = [
test for test in tests
if test.get('playbookID', '') in filtered_tests
]
tests_for_iteration = filter_tests_with_incompatible_version(
tests_for_iteration, server_numeric_version, prints_manager)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_names, modified_integrations_names = get_new_and_modified_integrations(
git_sha1)
if new_integrations_names:
print_warning('New Integrations Since Last Release:\n{}\n'.format(
'\n'.join(new_integrations_names)))
if modified_integrations_names:
print_warning('Updated Integrations Since Last Release:\n{}\n'.format(
'\n'.join(modified_integrations_names)))
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
integrations = get_integrations_for_test(test,
skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
print_warning('All Integrations for test "{}":'.format(
test.get('playbookID')))
print_warning(integrations_names)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names,
modified_integrations_names)
integrations_msg = '\n'.join([
'"{}" - {}'.format(key, val)
for key, val in integration_to_status.items()
])
print_warning('{}\n'.format(integrations_msg))
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
new_ints_params_set = set_integration_params(new_integrations,
secret_params,
instance_names_conf)
ints_to_configure_params_set = set_integration_params(
integrations_to_configure, secret_params, instance_names_conf)
if not new_ints_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(new_integrations)))
if not ints_to_configure_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(integrations_to_configure)))
if not (new_ints_params_set and ints_to_configure_params_set):
continue
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
print_warning(
'Start of Instance Testing ("Test" button) prior to Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Pre-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(clients))
# For each server url we install content
for thread_index, (client, server_url) in enumerate(zip(clients, servers)):
t = Thread(target=update_content_on_demisto_instance,
kwargs={
'client': client,
'server': server_url,
'prints_manager': threads_prints_manager,
'thread_index': thread_index
})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
new_integration_module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
print_warning(
'Start of Instance Testing ("Test" button) after the Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Post-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
__disable_integrations_instances(client, all_module_instances,
prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails,
new_integrations_names)
if not success:
sys.exit(1)