def test_downgrade_then_upgrade():
"""
If the patch version is greater than zero,
check that we can perform a version downgrade,
followed by version upgrade back to the current version
"""
helm_chart_path = os.environ.get("HELM_CHART_PATH")
if not helm_chart_path:
raise Exception(
"This test only works with HELM_CHART_PATH set to the path of the chart to be tested"
)
with open(os.path.join(git_root_dir, "Chart.yaml"), "r") as f:
astro_chart_dot_yaml = yaml.safe_load(f.read())
major, minor, patch = semver(astro_chart_dot_yaml['version']).release
if patch == 0:
print("This test is not applicable to patch version 0")
return
namespace = os.environ.get('NAMESPACE')
release_name = os.environ.get('RELEASE_NAME')
if not namespace:
print("NAMESPACE env var is not present, using 'astronomer' namespace")
namespace = 'astronomer'
if not release_name:
print(
"RELEASE_NAME env var is not present, assuming 'astronomer' is the release name"
)
release_name = 'astronomer'
config_path = os.path.join(git_root_dir, "upgrade_test_config.yaml")
# Get the existing values
check_output(
f"helm3 get values -n {namespace} {release_name} > {config_path}",
shell=True)
# attempt downgrade with the documented procedure
print("Performing patch version downgrade...")
command = \
"helm3 upgrade --reset-values " + \
f"-f {config_path} " + \
f"-n {namespace} " + \
f"--version={major}.{minor}.{patch - 1} " + \
f"{release_name} " + \
"astronomer-internal/astronomer"
print(command)
print(check_output(command, shell=True))
print("The downgrade worked, upgrading!")
command = \
"helm3 upgrade --reset-values " + \
f"-f {config_path} " + \
f"-n {namespace} " + \
f"--version={major}.{minor}.{patch} " + \
f"{release_name} " + \
helm_chart_path
print(command)
print(check_output(command, shell=True))
print("The upgrade worked!")
def test_downgrade_then_upgrade():
"""
If the patch version is greater than zero,
check that we can perform a version downgrade,
followed by version upgrade back to the current version
"""
helm_chart_path = getenv("HELM_CHART_PATH")
if not helm_chart_path:
raise Exception(
"This test only works with HELM_CHART_PATH set to the path of the chart to be tested"
)
with open(git_root_dir / "Chart.yaml") as f:
astro_chart_dot_yaml = yaml.safe_load(f.read())
major, minor, patch = semver(astro_chart_dot_yaml["version"]).release
if patch == 0:
print("This test is not applicable to patch version 0")
return
if not (namespace := getenv("NAMESPACE")):
print("NAMESPACE env var is not present, using 'astronomer' namespace")
namespace = "astronomer"
def test_chart_version_is_not_already_published():
"""
Tests that Chart.yaml has been incremented
"""
with open(os.path.join(git_root_dir, "Chart.yaml"), "r") as f:
astro_chart_dot_yaml = yaml.safe_load(f.read())
major, minor, patch = semver(astro_chart_dot_yaml['version']).release
if patch == 0:
print("This test does not apply to patch version zero")
return
repo_result = check_output(
f"helm3 search repo --output=json --version=^{major}.{minor} astronomer-internal/astronomer",
shell=True)
# Example result:
# [{"name":"astronomer-internal/astronomer","version":"0.19.3","app_version":"0.19.3","description":"Helm chart to deploy the Astronomer Platform"}]
repo_result = json.loads(repo_result)
for line in repo_result:
assert line['version'] != astro_chart_dot_yaml['version'], \
f"Version {astro_chart_dot_yaml['version']} is already released " + \
"to https://internal-helm.astronomer.io/, please increment 'version' " + \
"in both Chart.yaml and charts/astronomer/Chart.yaml"
from packaging.version import parse as semver
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def print_usage():
eprint(f"Description: {__doc__}\n")
eprint(f"Usage: {sys.argv[0]} <dir_to_check>\n")
if len(sys.argv) != 2:
print_usage()
sys.exit(1)
directory = Path(sys.argv[1]).absolute()
try:
with open(directory / "version.txt") as version_file:
version = version_file.read().strip()
except FileNotFoundError:
eprint(f"ERROR: version.txt not found in {directory}")
sys.exit(1)
if not semver(version).release:
eprint(f"ERROR: No valid semver found in {directory}/version.txt")
exit(1)
print(version)
def test_upgrade():
"""
Functional test for the LTS to LTS upgrade (0.16 to 0.23)
"""
with open(os.path.join(git_root_dir, "Chart.yaml"), "r") as f:
astro_chart_dot_yaml = yaml.safe_load(f.read())
major, minor, patch = semver(astro_chart_dot_yaml['version']).release
assert major == 0 and minor == 23, "This test is only applicable for 0.23"
upgrade_manifest_path = os.path.join(
git_root_dir,
"bin/migration-scripts/lts-to-lts/0.16-to-0.23/manifests/upgrade-0.16-to-0.23.yaml"
)
rollback_manifest_path = os.path.join(
git_root_dir,
"bin/migration-scripts/lts-to-lts/0.16-to-0.23/manifests/rollback-0.16-to-0.23.yaml"
)
namespace = os.environ.get('NAMESPACE')
release_name = os.environ.get('RELEASE_NAME')
if not namespace:
print("NAMESPACE env var is not present, using 'astronomer' namespace")
namespace = 'astronomer'
if not release_name:
print(
"RELEASE_NAME env var is not present, assuming 'astronomer' is the release name"
)
release_name = 'astronomer'
config_path = os.path.join(git_root_dir, "upgrade_test_config.yaml")
# Get the existing values
result = check_output(
f"helm3 history { release_name } -n { namespace } | tail -n 1",
shell=True).decode('utf8')
assert "0.16" in result
# rewrite manifest to replace with CI image for Kind test
with open(upgrade_manifest_path, "r") as f:
upgrade_manifest_data = f.read()
upgrade_manifest_data = upgrade_manifest_data.replace(
"image: quay.io/astronomer/lts-016-023-upgrade:latest",
"image: lts-016-023-upgrade:latest")
upgrade_manifest_data = upgrade_manifest_data.replace(
"imagePullPolicy: Always", "imagePullPolicy: Never")
with open(f"{upgrade_manifest_path}.test.yaml", "w") as f:
f.write(upgrade_manifest_data)
check_output(f"kubectl apply -f {upgrade_manifest_path}.test.yaml",
shell=True)
timeout = 800
start_time = time()
while True:
sleep(5)
result = check_output(
f"kubectl get pods -n default | grep upgrade-astronomer",
shell=True).decode('utf8')
if "Completed" in result:
print("Upgrade pod finished in success")
break
if "Error" in result:
print("Upgrade pod finished in error")
logs = check_output(
f"kubectl logs $(kubectl get pods -n default | grep upgrade-astronomer | awk "
+ "'{ print $1 }')",
shell=True).decode('utf8')
print(logs)
assert False, "Failed to perform upgrade, see logs"
if time() - start_time > timeout:
logs = check_output(
f"kubectl logs $(kubectl get pods -n default | grep upgrade-astronomer | awk "
+ "'{ print $1 }')",
shell=True).decode('utf8')
print(logs)
assert False, "Failed to perform upgrade (timeout!), see logs"
result = check_output(
f"helm3 history astronomer -n astronomer | tail -n 1",
shell=True).decode('utf8')
assert "0.23" in result and "deployed" in result, "Expected upgrade to be performed"
with open(rollback_manifest_path, "r") as f:
rollback_manifest_data = f.read()
rollback_manifest_data = rollback_manifest_data.replace(
"image: quay.io/astronomer/lts-016-023-upgrade:latest",
"image: lts-016-023-upgrade:latest")
rollback_manifest_data = rollback_manifest_data.replace(
"imagePullPolicy: Always", "imagePullPolicy: Never")
with open(f"{rollback_manifest_path}.test.yaml", "w") as f:
f.write(rollback_manifest_data)
check_output(f"kubectl apply -f {rollback_manifest_path}.test.yaml",
shell=True)
timeout = 800
start_time = time()
while True:
sleep(5)
result = check_output(f"kubectl get pods -n default | grep rollback",
shell=True).decode('utf8')
if "Completed" in result:
print("Rollback pod finished in success")
break
if "Error" in result:
print("Rollback pod finished in error")
logs = check_output(
f"kubectl logs $(kubectl get pods -n default | grep rollback | awk "
+ "'{ print $1 }')",
shell=True).decode('utf8')
print(logs)
assert False, "Failed to perform rollback, see logs"
if time() - start_time > timeout:
logs = check_output(
f"kubectl logs $(kubectl get pods -n default | grep rollback | awk "
+ "'{ print $1 }')",
shell=True).decode('utf8')
print(logs)
assert False, "Failed to perform rollback (timeout!), see logs"
result = check_output(
f"helm3 history astronomer -n astronomer | tail -n 1",
shell=True).decode('utf8')
assert "0.16" in result and "deployed" in result, "Expected rollback to be performed"
org = os.environ['GIT_ORG']
repository = os.environ['CIRCLE_PROJECT_REPONAME']
github = Github()
repo = github.get_repo(f'{ org }/{ repository }')
release_regex = re.compile("release-(\d*\.\d*)")
major_minor_version = release_regex.findall(branch)
most_recent_tag = None
if not len(major_minor_version):
raise Exception("ERROR: we are not on a release branch. Branch should be named release-X.Y where X and Y are positive integers")
major_minor_version = major_minor_version[0]
print(f"We are on a release branch: {branch}, detected major.minor version {major_minor_version}", file=sys.stderr)
print(f"We will find the most recent patch version of {major_minor_version} and return it incremented by one", file=sys.stderr)
for release in repo.get_releases():
version = semver(release.tag_name).release
this_major_minor = f"{version[0]}.{version[1]}"
if this_major_minor == major_minor_version:
most_recent_tag = release.tag_name
print(f"The most recent tag matching this release is {most_recent_tag}", file=sys.stderr)
break
if most_recent_tag:
base_version = semver(most_recent_tag).base_version
print(f"Parsed base version as {base_version}", file=sys.stderr)
base_version = semver(base_version)
major, minor, patch = base_version.release
patch += 1
new_version = ".".join(str(i) for i in [major, minor, patch])
else:
print("Did not detect a most recent version. Setting patch number to zero.", file=sys.stderr)
def test_upgrade():
"""
Functional test for the LTS to LTS upgrade (0.16 to 0.23)
"""
with open(Path(git_root_dir / "Chart.yaml"), "r") as f:
astro_chart_dot_yaml = yaml.safe_load(f.read())
major, minor, patch = semver(astro_chart_dot_yaml["version"]).release
assert major == 0 and minor == 23, "This test is only applicable for 0.23"
upgrade_manifest_path = Path(
git_root_dir /
"bin/migration-scripts/lts-to-lts/0.16-to-0.23/manifests/upgrade-0.16-to-0.23.yaml"
)
rollback_manifest_path = Path(
git_root_dir /
"bin/migration-scripts/lts-to-lts/0.16-to-0.23/manifests/rollback-0.16-to-0.23.yaml"
)
namespace = environ.get("NAMESPACE")
release_name = environ.get("RELEASE_NAME")
if not namespace:
print("NAMESPACE env var is not present, using 'astronomer' namespace")
namespace = "astronomer"
if not release_name:
print(
"RELEASE_NAME env var is not present, assuming 'astronomer' is the release name"
)
release_name = "astronomer"
# Get the existing values
result = check_output(
f"helm3 history { release_name } -n { namespace } | tail -n 1",
shell=True).decode("utf8")
assert "0.16" in result
# Rewrite some parts of the k8s manifest with testing-specific configs
with open(upgrade_manifest_path, "r") as f:
upgrade_manifest_data = f.read()
upgrade_manifest_data = upgrade_manifest_data.replace(
"image: quay.io/astronomer/lts-016-023-upgrade:latest",
"image: lts-016-023-upgrade:latest",
)
upgrade_manifest_data = upgrade_manifest_data.replace(
"imagePullPolicy: Always", "imagePullPolicy: Never")
upgrade_manifest_yaml = [
doc for doc in yaml.safe_load_all(upgrade_manifest_data)
]
for i, doc in enumerate(upgrade_manifest_yaml):
if doc.get("kind") == "Job":
try:
containers = doc["spec"]["template"]["spec"]["containers"]
for container in containers:
if not container.get("env"):
container["env"] = []
container["env"].append({
"name": "USE_INTERNAL_HELM_REPO",
"value": "True"
})
upgrade_manifest_yaml[i] = doc
except KeyError:
pass
upgrade_manifest_data = yaml.safe_dump_all(upgrade_manifest_yaml)
modified_upgrade_manifest_path = f"{upgrade_manifest_path}.test.yaml"
with open(modified_upgrade_manifest_path, "w") as f:
f.write(upgrade_manifest_data)
check_output(f"kubectl apply -f {modified_upgrade_manifest_path}",
shell=True)
timeout = 800
start_time = time()
while True:
sleep(5)
result = check_output(
"kubectl get pods -n default | grep upgrade-astronomer",
shell=True).decode("utf8")
if "Completed" in result:
print("Upgrade pod finished in success")
break
if "Error" in result:
print("Upgrade pod finished in error")
logs = check_output(
"kubectl logs $(kubectl get pods -n default | grep upgrade-astronomer | awk "
+ "'{ print $1 }')",
shell=True,
).decode("utf8")
print(logs)
assert False, "Failed to perform upgrade, see logs"
if time() - start_time > timeout:
logs = check_output(
"kubectl logs $(kubectl get pods -n default | grep upgrade-astronomer | awk "
+ "'{ print $1 }')",
shell=True,
).decode("utf8")
print(logs)
assert False, "Failed to perform upgrade (timeout!), see logs"
result = check_output("helm3 history astronomer -n astronomer | tail -n 1",
shell=True).decode("utf8")
assert "0.23" in result and "deployed" in result, "Expected upgrade to be performed"
with open(rollback_manifest_path, "r") as f:
rollback_manifest_data = f.read()
rollback_manifest_data = rollback_manifest_data.replace(
"image: quay.io/astronomer/lts-016-023-upgrade:latest",
"image: lts-016-023-upgrade:latest",
)
rollback_manifest_data = rollback_manifest_data.replace(
"imagePullPolicy: Always", "imagePullPolicy: Never")
with open(f"{rollback_manifest_path}.test.yaml", "w") as f:
f.write(rollback_manifest_data)
check_output(f"kubectl apply -f {rollback_manifest_path}.test.yaml",
shell=True)
timeout = 800
start_time = time()
while True:
sleep(5)
result = check_output("kubectl get pods -n default | grep rollback",
shell=True).decode("utf8")
if "Completed" in result:
print("Rollback pod finished in success")
break
if "Error" in result:
print("Rollback pod finished in error")
logs = check_output(
"kubectl logs $(kubectl get pods -n default | grep rollback | awk "
+ "'{ print $1 }')",
shell=True,
).decode("utf8")
print(logs)
assert False, "Failed to perform rollback, see logs"
if time() - start_time > timeout:
logs = check_output(
"kubectl logs $(kubectl get pods -n default | grep rollback | awk "
+ "'{ print $1 }')",
shell=True,
).decode("utf8")
print(logs)
assert False, "Failed to perform rollback (timeout!), see logs"
result = check_output("helm3 history astronomer -n astronomer | tail -n 1",
shell=True).decode("utf8")
assert ("0.16" in result
and "deployed" in result), "Expected rollback to be performed"