def main(
distro,
test_type,
from_version,
to_version,
python_package,
installer_script_url,
additional_packages=None,
destroy_node=False,
verbose=False,
):
# type: (str, str, str, str, str, str, str, bool, bool) -> None
# deployment objects for package files will be stored here.
file_upload_steps = []
# We always upload all the mock test configs from tests/ami/configs/ directory to a remote
# server.
# Those configs are used during various checks and tests. Uploading the configs is much less
# messy compared to manipulating the configs using sed on the server.
file_names = os.listdir(MOCK_CONFIGS_DIRECTORY)
for file_name in file_names:
config_file_path = os.path.join(MOCK_CONFIGS_DIRECTORY, file_name)
file_upload_step = _create_config_file_deployment_step(
config_file_path)
file_upload_steps.append(file_upload_step)
# Upload auxiliary files from tests/ami/files/
file_names = os.listdir(TEST_FILES_DIRECTORY)
for file_name in file_names:
file_path = os.path.join(TEST_FILES_DIRECTORY, file_name)
file_upload_step = _create_file_deployment_step(file_path, "ca_certs")
file_upload_steps.append(file_upload_step)
if test_type == "install":
install_package_source = to_version
else:
# install package is specified in from-version in case of upgrade
install_package_source = from_version
# prepare data for install_package
install_package_source_type = _get_source_type(install_package_source)
if install_package_source_type == "file":
# create install package file deployment object.
file_upload_steps.append(
_create_file_deployment_step(install_package_source,
"install_package"))
install_package_info = {
"type": install_package_source_type,
"source": install_package_source,
}
upgrade_package_info = None
# prepare data for upgrade_package if it is specified.
if test_type == "upgrade":
upgrade_package_source = to_version
upgrade_package_source_type = _get_source_type(upgrade_package_source)
if upgrade_package_source_type == "file":
# create install package file deployment object.
file_upload_steps.append(
_create_file_deployment_step(to_version, "upgrade_package"))
upgrade_package_info = {
"type": upgrade_package_source_type,
"source": upgrade_package_source,
}
distro_details = EC2_DISTRO_DETAILS_MAP[distro]
if distro.lower().startswith("windows"):
package_type = "windows"
script_extension = "ps1"
else:
package_type = ("deb" if distro.startswith("ubuntu")
or distro.startswith("debian") else "rpm")
script_extension = "sh"
script_filename = "test_%s.%s.j2" % (package_type, script_extension)
script_file_path = os.path.join(SCRIPTS_DIR, script_filename)
with open(script_file_path, "r") as fp:
script_content = fp.read()
cat_logs_script_file_path = os.path.join(
SCRIPTS_DIR, "cat_logs.%s" % (script_extension))
with open(cat_logs_script_file_path, "r") as fp:
cat_logs_script_content = fp.read()
installer_script_info = {
"source": installer_script_url or DEFAULT_INSTALLER_SCRIPT_URL
}
if os.path.exists(installer_script_url):
installer_script_info["type"] = "file"
file_upload_steps.append(
_create_file_deployment_step(installer_script_url,
"install-scalyr-agent-2"))
else:
if not _verify_url_exists(installer_script_url):
raise ValueError(
'Failed to retrieve installer script from "%s". Ensure that the URL is correct.'
% (installer_script_url))
installer_script_info["type"] = "url"
rendered_template = render_script_template(
script_template=script_content,
distro_name=distro,
distro_details=distro_details,
python_package=python_package,
test_type=test_type,
install_package=install_package_info,
upgrade_package=upgrade_package_info,
installer_script_url=installer_script_info,
additional_packages=additional_packages,
verbose=verbose,
)
# TODO: Lower those timeouts when upstream yum related issues or similar start to stabilize.
# All AMI tests should take less than 5 minutes, but in the last days (dec 1, 2020), they
# started to take 10 minutes with multiple timeouts.
if "windows" in distro.lower():
deploy_step_timeout = 440 # 320
deploy_overall_timeout = 460 # 320
cat_step_timeout = 10
max_tries = 3
else:
deploy_step_timeout = 320 # 260
deploy_overall_timeout = 340 # 280
max_tries = 3
cat_step_timeout = 5
remote_script_name = "deploy.{0}".format(script_extension)
test_package_step = ScriptDeployment(rendered_template,
name=remote_script_name,
timeout=deploy_step_timeout)
if file_upload_steps:
# Package files must be uploaded to the instance directly.
file_upload_steps.append(test_package_step) # type: ignore
deployment = MultiStepDeployment(add=file_upload_steps) # type: ignore
else:
deployment = MultiStepDeployment(add=test_package_step) # type: ignore
# Add a step which always cats agent.log file at the end. This helps us troubleshoot failures.
if "windows" not in distro.lower():
# NOTE: We don't add it on Windows since it tends to time out often
cat_logs_step = ScriptDeployment(cat_logs_script_content,
timeout=cat_step_timeout)
deployment.add(cat_logs_step)
else:
cat_logs_step = None # type: ignore
driver = get_libcloud_driver()
size = NodeSize(
distro_details["size_id"],
distro_details["size_id"],
0,
0,
0,
0,
driver,
)
image = NodeImage(distro_details["image_id"], distro_details["image_name"],
driver, None)
circle_branch_name = compat.os_environ_unicode.get("CIRCLE_BRANCH",
"unknown")
circle_branch_name = circle_branch_name.replace("/", "_").replace("-", "_")
circle_build_num = compat.os_environ_unicode.get("CIRCLE_BUILD_NUM",
random.randint(0, 1000))
name = "%s-automated-agent-tests-%s-branch-%s-build-%s" % (
distro,
test_type,
circle_branch_name,
circle_build_num,
)
print("Starting node provisioning and tests...")
start_time = int(time.time())
kwargs = {}
if destroy_node:
kwargs["at_exit_func"] = destroy_node_and_cleanup
try:
node = driver.deploy_node(name=name,
image=image,
size=size,
ssh_key=PRIVATE_KEY_PATH,
ex_keyname=KEY_NAME,
ex_security_groups=SECURITY_GROUPS,
ssh_username=distro_details["ssh_username"],
ssh_timeout=20,
max_tries=max_tries,
wait_period=15,
timeout=deploy_overall_timeout,
deploy=deployment,
**kwargs)
except DeploymentError as e:
print("Deployment failed: %s" % (str(e)))
node = e.node
success = False
test_package_step.exit_status = 1
stdout = getattr(e.original_error, "stdout", None)
stderr = getattr(e.original_error, "stderr", None)
else:
success = test_package_step.exit_status == 0
stdout = test_package_step.stdout
stderr = test_package_step.stderr
if cat_logs_step and cat_logs_step.stdout:
stdout += "\n" + cat_logs_step.stdout
if cat_logs_step and cat_logs_step.stderr:
stdout += "\n" + cat_logs_step.stderr
duration = int(time.time()) - start_time
if success:
print("Script successfully completed.")
else:
print("Script failed.")
print(("stdout: %s" % (stdout)))
print(("stderr: %s" % (stderr)))
print(("exit_code: %s" % (test_package_step.exit_status)))
print(("succeeded: %s" % (str(success))))
print(("duration: %s seconds" % (duration)))
# We don't destroy node if destroy_node is False (e.g. to aid with troubleshooting on failure
# and similar)
if node and destroy_node:
destroy_node_and_cleanup(driver=driver, node=node)
if not success:
sys.exit(1)
def main(
distro,
test_type,
from_version,
to_version,
python_package,
installer_script_url,
destroy_node=False,
verbose=False,
):
# type: (str, str, str, str, str, str, bool, bool) -> None
distro_details = EC2_DISTRO_DETAILS_MAP[distro]
if test_type == "install":
script_filename = "fresh_install_%s.sh.j2"
else:
script_filename = "upgrade_install_%s.sh.j2"
script_filename = script_filename % ("deb" if distro.startswith("ubuntu")
else "rpm")
script_file_path = os.path.join(SCRIPTS_DIR, script_filename)
with open(script_file_path, "r") as fp:
script_content = fp.read()
rendered_template = render_script_template(
script_template=script_content,
distro_details=distro_details,
python_package=python_package,
from_version=from_version,
to_version=to_version,
installer_script_url=installer_script_url,
verbose=verbose,
)
cls = get_driver(Provider.EC2)
driver = cls(ACCESS_KEY, SECRET_KEY, region=REGION)
size = NodeSize(
distro_details["size_id"],
distro_details["size_id"],
0,
0,
0,
0,
driver,
)
image = NodeImage(distro_details["image_id"], distro_details["image_name"],
driver, None)
name = "%s-automated-agent-%s-test-%s" % (
distro,
test_type,
random.randint(0, 1000),
)
step = ScriptDeployment(rendered_template, timeout=120)
print("Starting node provisioning and tests...")
start_time = int(time.time())
try:
node = driver.deploy_node(
name=name,
image=image,
size=size,
ssh_key=PRIVATE_KEY_PATH,
ex_keyname=KEY_NAME,
ex_security_groups=SECURITY_GROUPS,
ssh_username=distro_details["ssh_username"],
ssh_timeout=10,
timeout=140,
deploy=step,
at_exit_func=destroy_node_and_cleanup,
)
except DeploymentError as e:
print("Deployment failed: %s" % (str(e)))
node = e.node
success = False
step.exit_status = 1
stdout = getattr(e.original_error, "stdout", None)
stderr = getattr(e.original_error, "stderr", None)
else:
success = step.exit_status == 0
stdout = step.stdout
stderr = step.stderr
duration = int(time.time()) - start_time
if success:
print("Script successfully completed.")
else:
print("Script failed.")
print(("stdout: %s" % (stdout)))
print(("stderr: %s" % (stderr)))
print(("exit_code: %s" % (step.exit_status)))
print(("succeeded: %s" % (str(success))))
print(("duration: %s seconds" % (duration)))
# We don't destroy node if destroy_node is False (e.g. to aid with troubleshooting on failure
# and similar)
if node and destroy_node:
destroy_node_and_cleanup(driver=driver, node=node)
if not success:
sys.exit(1)
