def build_test(cloud='aws', distro='u2004', networking='kubenet', container_runtime='containerd', k8s_version='latest', kops_channel='alpha', kops_version=None, publish_version_marker=None, name_override=None, feature_flags=(), extra_flags=None, extra_dashboards=None, terraform_version=None, test_parallelism=25, test_timeout_minutes=60, skip_regex='', focus_regex=None, runs_per_day=0, scenario=None, env=None): # pylint: disable=too-many-statements,too-many-branches,too-many-arguments if kops_version is None: # TODO: Move to kops-ci/markers/master/ once validated kops_deploy_url = "https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt" elif kops_version.startswith("https://"): kops_deploy_url = kops_version kops_version = None else: kops_deploy_url = f"https://storage.googleapis.com/kops-ci/markers/release-{kops_version}/latest-ci-updown-green.txt" # pylint: disable=line-too-long # https://github.com/cilium/cilium/blob/71cfb265d53b63a2be3806fb3fd4425fa36262ff/Documentation/install/system_requirements.rst#centos-foot if networking in ("cilium", "cilium-etcd") and distro not in ["u2004", "u2004arm64", "deb10", "rhel8"]: # pylint: disable=line-too-long return None if should_skip_newer_k8s(k8s_version, kops_version): return None kops_image = distro_images[distro] kops_ssh_user = distros_ssh_user[distro] validation_wait = '20m' if distro == 'flatcar' else None marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info( k8s_version) args = create_args(kops_channel, networking, container_runtime, extra_flags, kops_image) suffix = "" if cloud and cloud != "aws": suffix += "-" + cloud if networking and networking != "kubenet": suffix += "-" + networking if distro: suffix += "-" + distro if k8s_version: suffix += "-k" + k8s_version.replace("1.", "") if kops_version: suffix += "-ko" + kops_version.replace("1.", "") if container_runtime: suffix += "-" + container_runtime tab = name_override or (f"kops-grid{suffix}") if tab in skip_jobs: return None job_name = f"e2e-{tab}" cron, runs_per_week = build_cron(tab, runs_per_day) # Scenario-specific parameters if env is None: env = {} tmpl_file = "periodic.yaml.jinja" if scenario is not None: tmpl_file = "periodic-scenario.yaml.jinja" name_hash = hashlib.md5(job_name.encode()).hexdigest() env['CLOUD_PROVIDER'] = cloud env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[11:16]}.test-cncf-aws.k8s.io" env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow' loader = jinja2.FileSystemLoader(searchpath="./templates") tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file) job = tmpl.render( job_name=job_name, cron=cron, kops_ssh_user=kops_ssh_user, create_args=args, k8s_deploy_url=k8s_deploy_url, kops_deploy_url=kops_deploy_url, test_parallelism=str(test_parallelism), job_timeout=str(test_timeout_minutes + 30) + 'm', test_timeout=str(test_timeout_minutes) + 'm', marker=marker, skip_regex=skip_regex, kops_feature_flags=','.join(feature_flags), terraform_version=terraform_version, test_package_bucket=test_package_bucket, test_package_dir=test_package_dir, focus_regex=focus_regex, publish_version_marker=publish_version_marker, validation_wait=validation_wait, image=image, scenario=scenario, env=env, ) spec = { 'cloud': cloud, 'networking': networking, 'distro': distro, 'k8s_version': k8s_version, 'kops_version': kops_version, 'container_runtime': container_runtime, 'kops_channel': kops_channel, } if feature_flags: spec['feature_flags'] = ','.join(feature_flags) if extra_flags: spec['extra_flags'] = ' '.join(extra_flags) jsonspec = json.dumps(spec, sort_keys=True) dashboards = [ 'sig-cluster-lifecycle-kops', 'google-aws', 'kops-kubetest2', f"kops-distro-{distro}", f"kops-k8s-{k8s_version or 'latest'}", f"kops-{kops_version or 'latest'}", ] if extra_dashboards: dashboards.extend(extra_dashboards) days_of_results = 90 if runs_per_week * days_of_results > 7500: # testgrid has a limit on number of test runs to show for a job days_of_results = math.floor(7500 / runs_per_week) annotations = { 'testgrid-dashboards': ', '.join(sorted(dashboards)), 'testgrid-days-of-results': str(days_of_results), 'testgrid-tab-name': tab, } for (k, v) in spec.items(): annotations[f"test.kops.k8s.io/{k}"] = v or "" extra = yaml.dump({'annotations': annotations}, width=9999, default_flow_style=False) output = f"\n# {jsonspec}\n{job.strip()}\n" for line in extra.splitlines(): output += f" {line}\n" return output, runs_per_week
def build_test(cloud='aws', distro='u2004', networking='kubenet', container_runtime='containerd', irsa=True, k8s_version='latest', kops_channel='alpha', kops_version=None, publish_version_marker=None, name_override=None, feature_flags=(), extra_flags=None, extra_dashboards=None, terraform_version=None, test_parallelism=25, test_timeout_minutes=60, skip_regex='', focus_regex=None, runs_per_day=0, scenario=None, env=None, template_path=None): # pylint: disable=too-many-statements,too-many-branches,too-many-arguments if kops_version is None: # TODO: Move to kops-ci/markers/master/ once validated kops_deploy_url = "https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt" elif kops_version.startswith("https://"): kops_deploy_url = kops_version kops_version = None else: kops_deploy_url = f"https://storage.googleapis.com/kops-ci/markers/release-{kops_version}/latest-ci-updown-green.txt" # pylint: disable=line-too-long # https://github.com/cilium/cilium/blob/f7a3f59fd74983c600bfce9cac364b76d20849d9/Documentation/operations/system_requirements.rst if networking in ("cilium", "cilium-etcd") and distro not in ["u2004", "u2004arm64", "deb10", "deb11", "rhel8", "amzn2"]: # pylint: disable=line-too-long return None if should_skip_newer_k8s(k8s_version, kops_version): return None if container_runtime == 'docker' and k8s_version not in ('1.21', '1.22', '1.23'): return None if cloud == 'aws': kops_image = distro_images[distro] kops_ssh_user = distros_ssh_user[distro] kops_ssh_key_path = '/etc/aws-ssh/aws-ssh-private' elif cloud == 'gce': kops_image = None kops_ssh_user = '******' kops_ssh_key_path = '/etc/ssh-key-secret/ssh-private' validation_wait = '20m' if distro == 'flatcar' else None suffix = "" if cloud and cloud != "aws": suffix += "-" + cloud if networking and networking != "kubenet": suffix += "-" + networking if distro: suffix += "-" + distro if k8s_version: suffix += "-k" + k8s_version.replace("1.", "") if kops_version: suffix += "-ko" + kops_version.replace("1.", "") if container_runtime: suffix += "-" + container_runtime tab = name_override or (f"kops-grid{suffix}") job_name = f"e2e-{tab}" if irsa and cloud == "aws" and scenario is None: if extra_flags is None: extra_flags = [] extra_flags.append("--discovery-store=s3://k8s-kops-prow/discovery") marker, k8s_deploy_url, test_package_bucket, test_package_dir = k8s_version_info( k8s_version) args = create_args(kops_channel, networking, container_runtime, extra_flags, kops_image) node_ig_overrides = "" cp_ig_overrides = "" if distro == "flatcar": # https://github.com/flatcar-linux/Flatcar/issues/220 node_ig_overrides += "spec.instanceMetadata.httpTokens=optional" cp_ig_overrides += "spec.instanceMetadata.httpTokens=optional" if tab in skip_jobs: return None cron, runs_per_week = build_cron(tab, runs_per_day) # Scenario-specific parameters if env is None: env = {} tmpl_file = "periodic.yaml.jinja" if scenario is not None: tmpl_file = "periodic-scenario.yaml.jinja" name_hash = hashlib.md5(job_name.encode()).hexdigest() env['CLOUD_PROVIDER'] = cloud env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[12:17]}.test-cncf-aws.k8s.io" env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow' env['KUBE_SSH_USER'] = kops_ssh_user if irsa and cloud == "aws": env['KOPS_IRSA'] = "true" tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file) job = tmpl.render( job_name=job_name, cloud=cloud, cron=cron, kops_ssh_user=kops_ssh_user, kops_ssh_key_path=kops_ssh_key_path, create_args=args, cp_ig_overrides=cp_ig_overrides, node_ig_overrides=node_ig_overrides, k8s_deploy_url=k8s_deploy_url, kops_deploy_url=kops_deploy_url, test_parallelism=str(test_parallelism), job_timeout=str(test_timeout_minutes + 30) + 'm', test_timeout=str(test_timeout_minutes) + 'm', marker=marker, template_path=template_path, skip_regex=skip_regex, kops_feature_flags=','.join(feature_flags), terraform_version=terraform_version, test_package_bucket=test_package_bucket, test_package_dir=test_package_dir, focus_regex=focus_regex, publish_version_marker=publish_version_marker, validation_wait=validation_wait, image=image, scenario=scenario, env=env, ) spec = { 'cloud': cloud, 'networking': networking, 'distro': distro, 'k8s_version': k8s_version, 'kops_version': kops_version, 'container_runtime': container_runtime, 'kops_channel': kops_channel, } if feature_flags: spec['feature_flags'] = ','.join(feature_flags) if extra_flags: spec['extra_flags'] = ' '.join(extra_flags) jsonspec = json.dumps(spec, sort_keys=True) dashboards = [ 'sig-cluster-lifecycle-kops', f"kops-distro-{distro}", f"kops-k8s-{k8s_version or 'latest'}", f"kops-{kops_version or 'latest'}", ] if cloud == 'aws': dashboards.extend(['google-aws']) if cloud == 'gce': dashboards.extend(['kops-gce']) if extra_dashboards: dashboards.extend(extra_dashboards) days_of_results = 90 if runs_per_week * days_of_results > 2000: # testgrid has a limit on number of test runs to show for a job days_of_results = math.floor(2000 / runs_per_week) annotations = { 'testgrid-dashboards': ', '.join(sorted(dashboards)), 'testgrid-days-of-results': str(days_of_results), 'testgrid-tab-name': tab, } for (k, v) in spec.items(): annotations[f"test.kops.k8s.io/{k}"] = v or "" extra = yaml.dump({'annotations': annotations}, width=9999, default_flow_style=False) output = f"\n# {jsonspec}\n{job.strip()}\n" for line in extra.splitlines(): output += f" {line}\n" return output, runs_per_week