def presubmit_test(branch='master',
cloud='aws',
distro='u2004',
networking='kubenet',
container_runtime='containerd',
k8s_version='latest',
kops_channel='alpha',
name=None,
tab_name=None,
feature_flags=(),
extra_flags=None,
extra_dashboards=None,
test_parallelism=25,
test_timeout_minutes=60,
skip_regex='',
focus_regex=None,
run_if_changed=None,
skip_report=False,
always_run=False,
scenario=None,
env=None):
# pylint: disable=too-many-statements,too-many-branches,too-many-arguments
if cloud == 'aws':
kops_image = distro_images[distro]
kops_ssh_user = distros_ssh_user[distro]
kops_ssh_key_path = '/etc/aws-ssh/aws-ssh-private'
elif cloud == 'gce':
kops_image = None
kops_ssh_user = '******'
kops_ssh_key_path = '/etc/ssh-key-secret/ssh-private'
marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info(
k8s_version)
args = create_args(kops_channel, networking, container_runtime,
extra_flags, kops_image)
# Scenario-specific parameters
if env is None:
env = {}
tmpl_file = "presubmit.yaml.jinja"
if scenario is not None:
tmpl_file = "presubmit-scenario.yaml.jinja"
name_hash = hashlib.md5(name.encode()).hexdigest()
env['CLOUD_PROVIDER'] = cloud
env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[11:16]}.test-cncf-aws.k8s.io"
env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow'
loader = jinja2.FileSystemLoader(searchpath="./templates")
tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file)
job = tmpl.render(
job_name=name,
branch=branch,
cloud=cloud,
kops_ssh_key_path=kops_ssh_key_path,
kops_ssh_user=kops_ssh_user,
create_args=args,
k8s_deploy_url=k8s_deploy_url,
test_parallelism=str(test_parallelism),
job_timeout=str(test_timeout_minutes + 30) + 'm',
test_timeout=str(test_timeout_minutes) + 'm',
marker=marker,
skip_regex=skip_regex,
kops_feature_flags=','.join(feature_flags),
test_package_bucket=test_package_bucket,
test_package_dir=test_package_dir,
focus_regex=focus_regex,
run_if_changed=run_if_changed,
skip_report='true' if skip_report else 'false',
always_run='true' if always_run else 'false',
image=image,
scenario=scenario,
env=env,
)
spec = {
'cloud': cloud,
'networking': networking,
'distro': distro,
'k8s_version': k8s_version,
'container_runtime': container_runtime,
'kops_channel': kops_channel,
}
if feature_flags:
spec['feature_flags'] = ','.join(feature_flags)
if extra_flags:
spec['extra_flags'] = ' '.join(extra_flags)
jsonspec = json.dumps(spec, sort_keys=True)
dashboards = [
'presubmits-kops',
'kops-presubmits',
'sig-cluster-lifecycle-kops',
'kops-kubetest2',
f"kops-distro-{distro}",
f"kops-k8s-{k8s_version or 'latest'}",
]
if extra_dashboards:
dashboards.extend(extra_dashboards)
annotations = {
'testgrid-dashboards': ', '.join(sorted(dashboards)),
'testgrid-days-of-results': '90',
'testgrid-tab-name': tab_name or name,
}
for (k, v) in spec.items():
annotations[f"test.kops.k8s.io/{k}"] = v or ""
extra = yaml.dump({'annotations': annotations},
width=9999,
default_flow_style=False)
output = f"\n# {jsonspec}{job}\n"
for line in extra.splitlines():
output += f" {line}\n"
return output
def build_test(cloud='aws',
distro='u2004',
networking='kubenet',
container_runtime='containerd',
k8s_version='latest',
kops_channel='alpha',
kops_version=None,
publish_version_marker=None,
name_override=None,
feature_flags=(),
extra_flags=None,
extra_dashboards=None,
terraform_version=None,
test_parallelism=25,
test_timeout_minutes=60,
skip_regex='',
focus_regex=None,
runs_per_day=0,
scenario=None,
env=None):
# pylint: disable=too-many-statements,too-many-branches,too-many-arguments
if kops_version is None:
# TODO: Move to kops-ci/markers/master/ once validated
kops_deploy_url = "https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt"
elif kops_version.startswith("https://"):
kops_deploy_url = kops_version
kops_version = None
else:
kops_deploy_url = f"https://storage.googleapis.com/kops-ci/markers/release-{kops_version}/latest-ci-updown-green.txt" # pylint: disable=line-too-long
# https://github.com/cilium/cilium/blob/71cfb265d53b63a2be3806fb3fd4425fa36262ff/Documentation/install/system_requirements.rst#centos-foot
if networking in ("cilium", "cilium-etcd") and distro not in ["u2004", "u2004arm64", "deb10", "rhel8"]: # pylint: disable=line-too-long
return None
if should_skip_newer_k8s(k8s_version, kops_version):
return None
kops_image = distro_images[distro]
kops_ssh_user = distros_ssh_user[distro]
validation_wait = '20m' if distro == 'flatcar' else None
marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info(
k8s_version)
args = create_args(kops_channel, networking, container_runtime,
extra_flags, kops_image)
suffix = ""
if cloud and cloud != "aws":
suffix += "-" + cloud
if networking and networking != "kubenet":
suffix += "-" + networking
if distro:
suffix += "-" + distro
if k8s_version:
suffix += "-k" + k8s_version.replace("1.", "")
if kops_version:
suffix += "-ko" + kops_version.replace("1.", "")
if container_runtime:
suffix += "-" + container_runtime
tab = name_override or (f"kops-grid{suffix}")
if tab in skip_jobs:
return None
job_name = f"e2e-{tab}"
cron, runs_per_week = build_cron(tab, runs_per_day)
# Scenario-specific parameters
if env is None:
env = {}
tmpl_file = "periodic.yaml.jinja"
if scenario is not None:
tmpl_file = "periodic-scenario.yaml.jinja"
name_hash = hashlib.md5(job_name.encode()).hexdigest()
env['CLOUD_PROVIDER'] = cloud
env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[11:16]}.test-cncf-aws.k8s.io"
env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow'
loader = jinja2.FileSystemLoader(searchpath="./templates")
tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file)
job = tmpl.render(
job_name=job_name,
cron=cron,
kops_ssh_user=kops_ssh_user,
create_args=args,
k8s_deploy_url=k8s_deploy_url,
kops_deploy_url=kops_deploy_url,
test_parallelism=str(test_parallelism),
job_timeout=str(test_timeout_minutes + 30) + 'm',
test_timeout=str(test_timeout_minutes) + 'm',
marker=marker,
skip_regex=skip_regex,
kops_feature_flags=','.join(feature_flags),
terraform_version=terraform_version,
test_package_bucket=test_package_bucket,
test_package_dir=test_package_dir,
focus_regex=focus_regex,
publish_version_marker=publish_version_marker,
validation_wait=validation_wait,
image=image,
scenario=scenario,
env=env,
)
spec = {
'cloud': cloud,
'networking': networking,
'distro': distro,
'k8s_version': k8s_version,
'kops_version': kops_version,
'container_runtime': container_runtime,
'kops_channel': kops_channel,
}
if feature_flags:
spec['feature_flags'] = ','.join(feature_flags)
if extra_flags:
spec['extra_flags'] = ' '.join(extra_flags)
jsonspec = json.dumps(spec, sort_keys=True)
dashboards = [
'sig-cluster-lifecycle-kops',
'google-aws',
'kops-kubetest2',
f"kops-distro-{distro}",
f"kops-k8s-{k8s_version or 'latest'}",
f"kops-{kops_version or 'latest'}",
]
if extra_dashboards:
dashboards.extend(extra_dashboards)
days_of_results = 90
if runs_per_week * days_of_results > 7500:
# testgrid has a limit on number of test runs to show for a job
days_of_results = math.floor(7500 / runs_per_week)
annotations = {
'testgrid-dashboards': ', '.join(sorted(dashboards)),
'testgrid-days-of-results': str(days_of_results),
'testgrid-tab-name': tab,
}
for (k, v) in spec.items():
annotations[f"test.kops.k8s.io/{k}"] = v or ""
extra = yaml.dump({'annotations': annotations},
width=9999,
default_flow_style=False)
output = f"\n# {jsonspec}\n{job.strip()}\n"
for line in extra.splitlines():
output += f" {line}\n"
return output, runs_per_week
def build_test(cloud='aws',
distro='u2004',
networking='kubenet',
container_runtime='containerd',
irsa=True,
k8s_version='latest',
kops_channel='alpha',
kops_version=None,
publish_version_marker=None,
name_override=None,
feature_flags=(),
extra_flags=None,
extra_dashboards=None,
terraform_version=None,
test_parallelism=25,
test_timeout_minutes=60,
skip_regex='',
focus_regex=None,
runs_per_day=0,
scenario=None,
env=None,
template_path=None):
# pylint: disable=too-many-statements,too-many-branches,too-many-arguments
if kops_version is None:
# TODO: Move to kops-ci/markers/master/ once validated
kops_deploy_url = "https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt"
elif kops_version.startswith("https://"):
kops_deploy_url = kops_version
kops_version = None
else:
kops_deploy_url = f"https://storage.googleapis.com/kops-ci/markers/release-{kops_version}/latest-ci-updown-green.txt" # pylint: disable=line-too-long
# https://github.com/cilium/cilium/blob/f7a3f59fd74983c600bfce9cac364b76d20849d9/Documentation/operations/system_requirements.rst
if networking in ("cilium", "cilium-etcd") and distro not in ["u2004", "u2004arm64", "deb10", "deb11", "rhel8", "amzn2"]: # pylint: disable=line-too-long
return None
if should_skip_newer_k8s(k8s_version, kops_version):
return None
if container_runtime == 'docker' and k8s_version not in ('1.21', '1.22',
'1.23'):
return None
if cloud == 'aws':
kops_image = distro_images[distro]
kops_ssh_user = distros_ssh_user[distro]
kops_ssh_key_path = '/etc/aws-ssh/aws-ssh-private'
elif cloud == 'gce':
kops_image = None
kops_ssh_user = '******'
kops_ssh_key_path = '/etc/ssh-key-secret/ssh-private'
validation_wait = '20m' if distro == 'flatcar' else None
suffix = ""
if cloud and cloud != "aws":
suffix += "-" + cloud
if networking and networking != "kubenet":
suffix += "-" + networking
if distro:
suffix += "-" + distro
if k8s_version:
suffix += "-k" + k8s_version.replace("1.", "")
if kops_version:
suffix += "-ko" + kops_version.replace("1.", "")
if container_runtime:
suffix += "-" + container_runtime
tab = name_override or (f"kops-grid{suffix}")
job_name = f"e2e-{tab}"
if irsa and cloud == "aws" and scenario is None:
if extra_flags is None:
extra_flags = []
extra_flags.append("--discovery-store=s3://k8s-kops-prow/discovery")
marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info(
k8s_version)
args = create_args(kops_channel, networking, container_runtime,
extra_flags, kops_image)
node_ig_overrides = ""
cp_ig_overrides = ""
if distro == "flatcar":
# https://github.com/flatcar-linux/Flatcar/issues/220
node_ig_overrides += "spec.instanceMetadata.httpTokens=optional"
cp_ig_overrides += "spec.instanceMetadata.httpTokens=optional"
if tab in skip_jobs:
return None
cron, runs_per_week = build_cron(tab, runs_per_day)
# Scenario-specific parameters
if env is None:
env = {}
tmpl_file = "periodic.yaml.jinja"
if scenario is not None:
tmpl_file = "periodic-scenario.yaml.jinja"
name_hash = hashlib.md5(job_name.encode()).hexdigest()
env['CLOUD_PROVIDER'] = cloud
env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[12:17]}.test-cncf-aws.k8s.io"
env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow'
env['KUBE_SSH_USER'] = kops_ssh_user
if irsa and cloud == "aws":
env['KOPS_IRSA'] = "true"
tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file)
job = tmpl.render(
job_name=job_name,
cloud=cloud,
cron=cron,
kops_ssh_user=kops_ssh_user,
kops_ssh_key_path=kops_ssh_key_path,
create_args=args,
cp_ig_overrides=cp_ig_overrides,
node_ig_overrides=node_ig_overrides,
k8s_deploy_url=k8s_deploy_url,
kops_deploy_url=kops_deploy_url,
test_parallelism=str(test_parallelism),
job_timeout=str(test_timeout_minutes + 30) + 'm',
test_timeout=str(test_timeout_minutes) + 'm',
marker=marker,
template_path=template_path,
skip_regex=skip_regex,
kops_feature_flags=','.join(feature_flags),
terraform_version=terraform_version,
test_package_bucket=test_package_bucket,
test_package_dir=test_package_dir,
focus_regex=focus_regex,
publish_version_marker=publish_version_marker,
validation_wait=validation_wait,
image=image,
scenario=scenario,
env=env,
)
spec = {
'cloud': cloud,
'networking': networking,
'distro': distro,
'k8s_version': k8s_version,
'kops_version': kops_version,
'container_runtime': container_runtime,
'kops_channel': kops_channel,
}
if feature_flags:
spec['feature_flags'] = ','.join(feature_flags)
if extra_flags:
spec['extra_flags'] = ' '.join(extra_flags)
jsonspec = json.dumps(spec, sort_keys=True)
dashboards = [
'sig-cluster-lifecycle-kops',
f"kops-distro-{distro}",
f"kops-k8s-{k8s_version or 'latest'}",
f"kops-{kops_version or 'latest'}",
]
if cloud == 'aws':
dashboards.extend(['google-aws'])
if cloud == 'gce':
dashboards.extend(['kops-gce'])
if extra_dashboards:
dashboards.extend(extra_dashboards)
days_of_results = 90
if runs_per_week * days_of_results > 2000:
# testgrid has a limit on number of test runs to show for a job
days_of_results = math.floor(2000 / runs_per_week)
annotations = {
'testgrid-dashboards': ', '.join(sorted(dashboards)),
'testgrid-days-of-results': str(days_of_results),
'testgrid-tab-name': tab,
}
for (k, v) in spec.items():
annotations[f"test.kops.k8s.io/{k}"] = v or ""
extra = yaml.dump({'annotations': annotations},
width=9999,
default_flow_style=False)
output = f"\n# {jsonspec}\n{job.strip()}\n"
for line in extra.splitlines():
output += f" {line}\n"
return output, runs_per_week