def main( distro, test_type, from_version, to_version, python_package, installer_script_url, additional_packages=None, destroy_node=False, verbose=False, ): # type: (str, str, str, str, str, str, str, bool, bool) -> None # deployment objects for package files will be stored here. file_upload_steps = [] # We always upload all the mock test configs from tests/ami/configs/ directory to a remote # server. # Those configs are used during various checks and tests. Uploading the configs is much less # messy compared to manipulating the configs using sed on the server. file_names = os.listdir(MOCK_CONFIGS_DIRECTORY) for file_name in file_names: config_file_path = os.path.join(MOCK_CONFIGS_DIRECTORY, file_name) file_upload_step = _create_config_file_deployment_step( config_file_path) file_upload_steps.append(file_upload_step) # Upload auxiliary files from tests/ami/files/ file_names = os.listdir(TEST_FILES_DIRECTORY) for file_name in file_names: file_path = os.path.join(TEST_FILES_DIRECTORY, file_name) file_upload_step = _create_file_deployment_step(file_path, "ca_certs") file_upload_steps.append(file_upload_step) if test_type == "install": install_package_source = to_version else: # install package is specified in from-version in case of upgrade install_package_source = from_version # prepare data for install_package install_package_source_type = _get_source_type(install_package_source) if install_package_source_type == "file": # create install package file deployment object. file_upload_steps.append( _create_file_deployment_step(install_package_source, "install_package")) install_package_info = { "type": install_package_source_type, "source": install_package_source, } upgrade_package_info = None # prepare data for upgrade_package if it is specified. if test_type == "upgrade": upgrade_package_source = to_version upgrade_package_source_type = _get_source_type(upgrade_package_source) if upgrade_package_source_type == "file": # create install package file deployment object. file_upload_steps.append( _create_file_deployment_step(to_version, "upgrade_package")) upgrade_package_info = { "type": upgrade_package_source_type, "source": upgrade_package_source, } distro_details = EC2_DISTRO_DETAILS_MAP[distro] if distro.lower().startswith("windows"): package_type = "windows" script_extension = "ps1" else: package_type = ("deb" if distro.startswith("ubuntu") or distro.startswith("debian") else "rpm") script_extension = "sh" script_filename = "test_%s.%s.j2" % (package_type, script_extension) script_file_path = os.path.join(SCRIPTS_DIR, script_filename) with open(script_file_path, "r") as fp: script_content = fp.read() cat_logs_script_file_path = os.path.join( SCRIPTS_DIR, "cat_logs.%s" % (script_extension)) with open(cat_logs_script_file_path, "r") as fp: cat_logs_script_content = fp.read() installer_script_info = { "source": installer_script_url or DEFAULT_INSTALLER_SCRIPT_URL } if os.path.exists(installer_script_url): installer_script_info["type"] = "file" file_upload_steps.append( _create_file_deployment_step(installer_script_url, "install-scalyr-agent-2")) else: if not _verify_url_exists(installer_script_url): raise ValueError( 'Failed to retrieve installer script from "%s". Ensure that the URL is correct.' % (installer_script_url)) installer_script_info["type"] = "url" rendered_template = render_script_template( script_template=script_content, distro_name=distro, distro_details=distro_details, python_package=python_package, test_type=test_type, install_package=install_package_info, upgrade_package=upgrade_package_info, installer_script_url=installer_script_info, additional_packages=additional_packages, verbose=verbose, ) # TODO: Lower those timeouts when upstream yum related issues or similar start to stabilize. # All AMI tests should take less than 5 minutes, but in the last days (dec 1, 2020), they # started to take 10 minutes with multiple timeouts. if "windows" in distro.lower(): deploy_step_timeout = 440 # 320 deploy_overall_timeout = 460 # 320 cat_step_timeout = 10 max_tries = 3 else: deploy_step_timeout = 320 # 260 deploy_overall_timeout = 340 # 280 max_tries = 3 cat_step_timeout = 5 remote_script_name = "deploy.{0}".format(script_extension) test_package_step = ScriptDeployment(rendered_template, name=remote_script_name, timeout=deploy_step_timeout) if file_upload_steps: # Package files must be uploaded to the instance directly. file_upload_steps.append(test_package_step) # type: ignore deployment = MultiStepDeployment(add=file_upload_steps) # type: ignore else: deployment = MultiStepDeployment(add=test_package_step) # type: ignore # Add a step which always cats agent.log file at the end. This helps us troubleshoot failures. if "windows" not in distro.lower(): # NOTE: We don't add it on Windows since it tends to time out often cat_logs_step = ScriptDeployment(cat_logs_script_content, timeout=cat_step_timeout) deployment.add(cat_logs_step) else: cat_logs_step = None # type: ignore driver = get_libcloud_driver() size = NodeSize( distro_details["size_id"], distro_details["size_id"], 0, 0, 0, 0, driver, ) image = NodeImage(distro_details["image_id"], distro_details["image_name"], driver, None) circle_branch_name = compat.os_environ_unicode.get("CIRCLE_BRANCH", "unknown") circle_branch_name = circle_branch_name.replace("/", "_").replace("-", "_") circle_build_num = compat.os_environ_unicode.get("CIRCLE_BUILD_NUM", random.randint(0, 1000)) name = "%s-automated-agent-tests-%s-branch-%s-build-%s" % ( distro, test_type, circle_branch_name, circle_build_num, ) print("Starting node provisioning and tests...") start_time = int(time.time()) kwargs = {} if destroy_node: kwargs["at_exit_func"] = destroy_node_and_cleanup try: node = driver.deploy_node(name=name, image=image, size=size, ssh_key=PRIVATE_KEY_PATH, ex_keyname=KEY_NAME, ex_security_groups=SECURITY_GROUPS, ssh_username=distro_details["ssh_username"], ssh_timeout=20, max_tries=max_tries, wait_period=15, timeout=deploy_overall_timeout, deploy=deployment, **kwargs) except DeploymentError as e: print("Deployment failed: %s" % (str(e))) node = e.node success = False test_package_step.exit_status = 1 stdout = getattr(e.original_error, "stdout", None) stderr = getattr(e.original_error, "stderr", None) else: success = test_package_step.exit_status == 0 stdout = test_package_step.stdout stderr = test_package_step.stderr if cat_logs_step and cat_logs_step.stdout: stdout += "\n" + cat_logs_step.stdout if cat_logs_step and cat_logs_step.stderr: stdout += "\n" + cat_logs_step.stderr duration = int(time.time()) - start_time if success: print("Script successfully completed.") else: print("Script failed.") print(("stdout: %s" % (stdout))) print(("stderr: %s" % (stderr))) print(("exit_code: %s" % (test_package_step.exit_status))) print(("succeeded: %s" % (str(success)))) print(("duration: %s seconds" % (duration))) # We don't destroy node if destroy_node is False (e.g. to aid with troubleshooting on failure # and similar) if node and destroy_node: destroy_node_and_cleanup(driver=driver, node=node) if not success: sys.exit(1)
def main( distro, test_type, from_version, to_version, python_package, installer_script_url, destroy_node=False, verbose=False, ): # type: (str, str, str, str, str, str, bool, bool) -> None distro_details = EC2_DISTRO_DETAILS_MAP[distro] if test_type == "install": script_filename = "fresh_install_%s.sh.j2" else: script_filename = "upgrade_install_%s.sh.j2" script_filename = script_filename % ("deb" if distro.startswith("ubuntu") else "rpm") script_file_path = os.path.join(SCRIPTS_DIR, script_filename) with open(script_file_path, "r") as fp: script_content = fp.read() rendered_template = render_script_template( script_template=script_content, distro_details=distro_details, python_package=python_package, from_version=from_version, to_version=to_version, installer_script_url=installer_script_url, verbose=verbose, ) cls = get_driver(Provider.EC2) driver = cls(ACCESS_KEY, SECRET_KEY, region=REGION) size = NodeSize( distro_details["size_id"], distro_details["size_id"], 0, 0, 0, 0, driver, ) image = NodeImage(distro_details["image_id"], distro_details["image_name"], driver, None) name = "%s-automated-agent-%s-test-%s" % ( distro, test_type, random.randint(0, 1000), ) step = ScriptDeployment(rendered_template, timeout=120) print("Starting node provisioning and tests...") start_time = int(time.time()) try: node = driver.deploy_node( name=name, image=image, size=size, ssh_key=PRIVATE_KEY_PATH, ex_keyname=KEY_NAME, ex_security_groups=SECURITY_GROUPS, ssh_username=distro_details["ssh_username"], ssh_timeout=10, timeout=140, deploy=step, at_exit_func=destroy_node_and_cleanup, ) except DeploymentError as e: print("Deployment failed: %s" % (str(e))) node = e.node success = False step.exit_status = 1 stdout = getattr(e.original_error, "stdout", None) stderr = getattr(e.original_error, "stderr", None) else: success = step.exit_status == 0 stdout = step.stdout stderr = step.stderr duration = int(time.time()) - start_time if success: print("Script successfully completed.") else: print("Script failed.") print(("stdout: %s" % (stdout))) print(("stderr: %s" % (stderr))) print(("exit_code: %s" % (step.exit_status))) print(("succeeded: %s" % (str(success)))) print(("duration: %s seconds" % (duration))) # We don't destroy node if destroy_node is False (e.g. to aid with troubleshooting on failure # and similar) if node and destroy_node: destroy_node_and_cleanup(driver=driver, node=node) if not success: sys.exit(1)