def create_started_file(bucket): """Create the started file in gcs for gubernator.""" contents = prow_artifacts.create_started() target = os.path.join(prow_artifacts.get_gcs_dir(bucket), "started.json") util.upload_to_gcs(contents, target)
def run_papermill_job( notebook_path, name, namespace, # pylint: disable=too-many-branches,too-many-statements repos, image): """Generate a K8s job to run a notebook using papermill Args: notebook_path: Path to the notebook. This should be in the form "{REPO_OWNER}/{REPO}/path/to/notebook.ipynb" name: Name for the K8s job namespace: The namespace where the job should run. repos: Which repos to checkout; if None or empty tries to infer based on PROW environment variables image: The docker image to run the notebook in. """ util.maybe_activate_service_account() with open("job.yaml") as hf: job = yaml.load(hf) if notebook_path.startswith("/"): raise ValueError( "notebook_path={0} should not start with /".format(notebook_path)) # We need to checkout the correct version of the code # in presubmits and postsubmits. We should check the environment variables # for the prow environment variables to get the appropriate values. # We should probably also only do that if the # See # https://github.com/kubernetes/test-infra/blob/45246b09ed105698aa8fb928b7736d14480def29/prow/jobs.md#job-environment-variables if not repos: repos = argo_build_util.get_repo_from_prow_env() if not repos: raise ValueError("Could not get repos from prow environment variable " "and --repos isn't explicitly set") repos += ",kubeflow/testing@HEAD" logging.info("Repos set to %s", repos) job["spec"]["template"]["spec"]["initContainers"][0]["command"] = [ "/usr/local/bin/checkout_repos.sh", "--repos=" + repos, "--src_dir=/src", "--depth=all", ] job["spec"]["template"]["spec"]["containers"][0]["image"] = image full_notebook_path = os.path.join("/src", notebook_path) job["spec"]["template"]["spec"]["containers"][0]["command"] = [ "python3", "-m", "kubeflow.examples.notebook_tests.execute_notebook", "--notebook_path", full_notebook_path ] job["spec"]["template"]["spec"]["containers"][0][ "workingDir"] = os.path.dirname(full_notebook_path) # The prow bucket to use for results/artifacts prow_bucket = prow_artifacts.PROW_RESULTS_BUCKET if os.getenv("REPO_OWNER") and os.getenv("REPO_NAME"): # Running under prow prow_dir = prow_artifacts.get_gcs_dir(prow_bucket) logging.info("Prow artifacts dir: %s", prow_dir) prow_dir = os.path.join(prow_dir, "artifacts") if os.getenv("TEST_TARGET_NAME"): prow_dir = os.path.join(prow_dir, os.getenv("TEST_TARGET_NAME").lstrip("/")) prow_bucket, prow_path = util.split_gcs_uri(prow_dir) else: prow_path = "notebook-test" + datetime.datetime.now().strftime( "%H%M%S") prow_path = prow_path + "-" + uuid.uuid4().hex[0:3] prow_dir = util.to_gcs_uri(prow_bucket, prow_path) prow_path = os.path.join(prow_path, name + ".html") output_gcs = util.to_gcs_uri(NB_BUCKET, prow_path) job["spec"]["template"]["spec"]["containers"][0]["env"] = [ { "name": "OUTPUT_GCS", "value": output_gcs }, { "name": "PYTHONPATH", "value": "/src/kubeflow/testing/py:/src/kubeflow/examples/py" }, ] logging.info("Notebook will be written to %s", output_gcs) util.load_kube_config(persist_config=False) if name: job["metadata"]["name"] = name else: job["metadata"]["name"] = ("notebook-test-" + datetime.datetime.now().strftime("%H%M%S") + "-" + uuid.uuid4().hex[0:3]) name = job["metadata"]["name"] job["metadata"]["namespace"] = namespace # Create an API client object to talk to the K8s master. api_client = k8s_client.ApiClient() batch_api = k8s_client.BatchV1Api(api_client) logging.info("Creating job:\n%s", yaml.dump(job)) actual_job = batch_api.create_namespaced_job(job["metadata"]["namespace"], job) logging.info("Created job %s.%s:\n%s", namespace, name, yaml.safe_dump(actual_job.to_dict())) final_job = util.wait_for_job(api_client, namespace, name, timeout=datetime.timedelta(minutes=30)) logging.info("Final job:\n%s", yaml.safe_dump(final_job.to_dict())) # Download notebook html to artifacts logging.info("Copying %s to bucket %s", output_gcs, prow_bucket) storage_client = storage.Client() bucket = storage_client.get_bucket(NB_BUCKET) blob = bucket.get_blob(prow_path) destination_bucket = storage_client.get_bucket(prow_bucket) bucket.copy_blob(blob, destination_bucket) if not final_job.status.conditions: raise RuntimeError("Job {0}.{1}; did not complete".format( namespace, name)) last_condition = final_job.status.conditions[-1] if last_condition.type not in ["Complete"]: logging.error("Job didn't complete successfully") raise RuntimeError("Job {0}.{1} failed".format(namespace, name))
def run(args, file_handler): # pylint: disable=too-many-statements,too-many-branches job_type = os.getenv("JOB_TYPE") repo_owner = os.getenv("REPO_OWNER") repo_name = os.getenv("REPO_NAME") pull_base_sha = os.getenv("PULL_BASE_SHA") # For presubmit/postsubmit jobs, find the list of files changed by the PR. diff_command = [] if job_type == "presubmit": diff_command = ["git", "diff", "--name-only", "master"] elif job_type == "postsubmit": diff_command = ["git", "diff", "--name-only", pull_base_sha + "^", pull_base_sha] changed_files = [] if job_type == "presubmit" or job_type == "postsubmit": changed_files = util.run(diff_command, cwd=os.path.join(args.repos_dir, repo_owner, repo_name)).splitlines() for f in changed_files: logging.info("File %s is modified.", f) if args.release: generate_env_from_head(args) workflows = [] if args.config_file: workflows.extend(parse_config_file(args.config_file, args.repos_dir)) create_started_file(args.bucket) util.maybe_activate_service_account() util.configure_kubectl(args.project, args.zone, args.cluster) util.load_kube_config() workflow_names = [] ui_urls = {} for w in workflows: # Create the name for the workflow # We truncate sha numbers to prevent the workflow name from being too large. # Workflow name should not be more than 63 characters because its used # as a label on the pods. workflow_name = os.getenv("JOB_NAME") + "-" + w.name ks_cmd = get_ksonnet_cmd(w) # Print ksonnet version util.run([ks_cmd, "version"]) # Skip this workflow if it is scoped to a different job type. if w.job_types and not job_type in w.job_types: logging.info("Skipping workflow %s because job type %s is not one of " "%s.", w.name, job_type, w.job_types) continue # If we are scoping this workflow to specific directories, check if any files # modified match the specified regex patterns. dir_modified = False if w.include_dirs: for f in changed_files: for d in w.include_dirs: if fnmatch.fnmatch(f, d): dir_modified = True logging.info("Triggering workflow %s because %s in dir %s is modified.", w.name, f, d) break if dir_modified: break # Only consider modified files when the job is pre or post submit, and if # the include_dirs stanza is defined. if job_type != "periodic" and w.include_dirs and not dir_modified: logging.info("Skipping workflow %s because no code modified in %s.", w.name, w.include_dirs) continue if job_type == "presubmit": workflow_name += "-{0}".format(os.getenv("PULL_NUMBER")) workflow_name += "-{0}".format(os.getenv("PULL_PULL_SHA")[0:7]) elif job_type == "postsubmit": workflow_name += "-{0}".format(os.getenv("PULL_BASE_SHA")[0:7]) workflow_name += "-{0}".format(os.getenv("BUILD_NUMBER")) salt = uuid.uuid4().hex[0:4] # Add some salt. This is mostly a convenience for the case where you # are submitting jobs manually for testing/debugging. Since the prow should # vend unique build numbers for each job. workflow_name += "-{0}".format(salt) workflow_names.append(workflow_name) # Create a new environment for this run env = workflow_name util.run([ks_cmd, "env", "add", env], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "name", workflow_name], cwd=w.app_dir) # Set the prow environment variables. prow_env = [] names = ["JOB_NAME", "JOB_TYPE", "BUILD_ID", "BUILD_NUMBER", "PULL_BASE_SHA", "PULL_NUMBER", "PULL_PULL_SHA", "REPO_OWNER", "REPO_NAME"] names.sort() for v in names: if not os.getenv(v): continue prow_env.append("{0}={1}".format(v, os.getenv(v))) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "prow_env", ",".join(prow_env)], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "namespace", get_namespace(args)], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "bucket", args.bucket], cwd=w.app_dir) if args.release: util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "versionTag", os.getenv("VERSION_TAG")], cwd=w.app_dir) # Set any extra params. We do this in alphabetical order to make it easier to verify in # the unittest. param_names = w.params.keys() param_names.sort() for k in param_names: util.run([ks_cmd, "param", "set", "--env=" + env, w.component, k, "{0}".format(w.params[k])], cwd=w.app_dir) # For debugging print out the manifest util.run([ks_cmd, "show", env, "-c", w.component], cwd=w.app_dir) util.run([ks_cmd, "apply", env, "-c", w.component], cwd=w.app_dir) ui_url = ("http://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/{0}" "?tab=workflow".format(workflow_name)) ui_urls[workflow_name] = ui_url logging.info("URL for workflow: %s", ui_url) success = True workflow_phase = {} try: results = argo_client.wait_for_workflows(get_namespace(args), workflow_names, timeout=datetime.timedelta(minutes=180), status_callback=argo_client.log_status) for r in results: phase = r.get("status", {}).get("phase") name = r.get("metadata", {}).get("name") workflow_phase[name] = phase if phase != "Succeeded": success = False logging.info("Workflow %s/%s finished phase: %s", get_namespace(args), name, phase) except util.TimeoutError: success = False logging.exception("Time out waiting for Workflows %s to finish", ",".join(workflow_names)) except Exception as e: # We explicitly log any exceptions so that they will be captured in the # build-log.txt that is uploaded to Gubernator. logging.exception("Exception occurred: %s", e) raise finally: success = prow_artifacts.finalize_prow_job(args.bucket, success, workflow_phase, ui_urls) # Upload logs to GCS. No logs after this point will appear in the # file in gcs file_handler.flush() util.upload_file_to_gcs( file_handler.baseFilename, os.path.join(prow_artifacts.get_gcs_dir(args.bucket), "build-log.txt")) return success
def load_tekton_run( params, test_target_name, tekton_run, # pylint: disable=too-many-branches bucket, repo_owner, repo_under_test, pull_revision): """Load Tekton configs and override information from Prow. Args: params: Extra parameters to be passed into Tekton pipelines. test_target_name: test target name as classname in JUNIT. tekton_run: File path to the PipelineRun config. bucket: GCS bucket to write artifacts to. """ with open(tekton_run) as f: config = yaml.load(f) if config.get("kind", "") != "PipelineRun": raise ValueError("Invalid config (not PipelineRun): " + config) if not "generateName" in config["metadata"]: raise ValueError("TektonPipeline is missing generateName") logging.info("Reading Tekton PipelineRun config: %s", config["metadata"]["generateName"]) workflow_name = config["metadata"]["generateName"] artifacts_gcs = prow_artifacts.get_gcs_dir(bucket) junit_path = "artifacts/junit_{run_name}".format(run_name=workflow_name) args = { "test-target-name": test_target_name, "artifacts-gcs": artifacts_gcs, "junit-path": junit_path, } for p in params: args[p["name"]] = p["value"] for param in config.get("spec", {}).get("params", []): n = param.get("name", "") v = param.get("value", "") if n and v and not n in args: args[n] = v config["spec"]["params"] = [] for n in args: logging.info("Writing Tekton param: %s -> %s", n, args[n]) config["spec"]["params"].append({ "name": n, "value": args[n], }) # Points to the revision under test. repo_url = "https://github.com/{owner}/{name}.git".format( owner=repo_owner, name=repo_under_test) replacing_param = [ { "name": "url", "value": repo_url }, { "name": "revision", "value": pull_revision }, ] job_type = os.getenv("JOB_TYPE", "").lower() if job_type in ["presubmit", "postsubmit"]: logging.info("Job is type %s; looking for url %s", job_type, repo_url) foundRepo = False for resource in config["spec"].get("resources", []): if resource.get("resourceSpec", {}).get("type", "") != "git": pass for param in resource.get("resourceSpec", {}).get("params", []): if param.get("name", "") != "url": continue if param.get("value", "") == repo_url: foundRepo = True resource["resourceSpec"]["params"] = replacing_param break if not foundRepo: raise ValueError( ("The TektonPipelineRun is missing a pipeline git " "resource that matches the repo being tested by " "prow. The pipeline parameters must include " "a git resource whose URL is {0}".format(repo_url))) else: logging.info("Job is type %s; not looking for repo", job_type) return config
def push_kfctl_to_gcs(kfctl_path): bucket = prow_artifacts.PROW_RESULTS_BUCKET logging.info("Bucket name: %s", prow_artifacts.get_gcs_dir(bucket)) gcs_path = os.path.join( prow_artifacts.get_gcs_dir(bucket) + "/artifacts/build_bin/kfctl") util.upload_file_to_gcs(kfctl_path, gcs_path)
def run(args, file_handler): # pylint: disable=too-many-statements,too-many-branches # Check https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md # for a description of the injected environment variables. job_type = os.getenv("JOB_TYPE") repo_owner = os.getenv("REPO_OWNER") repo_name = os.getenv("REPO_NAME") base_branch_name = os.getenv("PULL_BASE_REF") pull_base_sha = os.getenv("PULL_BASE_SHA") # For presubmit/postsubmit jobs, find the list of files changed by the PR. diff_command = [] if job_type == "presubmit": # We need to get a common ancestor for the PR and the base branch cloned_repo_dir = os.path.join(args.repos_dir, repo_owner, repo_name) _ = util.run([ "git", "fetch", "origin", base_branch_name + ":refs/remotes/origin/" + base_branch_name ], cwd=cloned_repo_dir) diff_command = ["git", "diff", "--name-only"] diff_branch = "remotes/origin/{}".format(base_branch_name) try: common_ancestor = util.run( ["git", "merge-base", "HEAD", diff_branch], cwd=cloned_repo_dir) diff_command.append(common_ancestor) except subprocess.CalledProcessError as e: logging.warning( "git merge-base failed; see " "https://github.com/kubeflow/kubeflow/issues/3523. Diff " "will be computed against the current master and " "therefore files not changed in the PR might be " "considered when determining which tests to trigger") diff_command.append(diff_branch) elif job_type == "postsubmit": # See: https://git-scm.com/docs/git-diff # This syntax compares the commit before pull_base_sha with the commit # at pull_base_sha diff_command = [ "git", "diff", "--name-only", pull_base_sha + "^", pull_base_sha ] changed_files = [] if job_type in ("presubmit", "postsubmit"): changed_files = util.run(diff_command, cwd=os.path.join(args.repos_dir, repo_owner, repo_name)).splitlines() for f in changed_files: logging.info("File %s is modified.", f) if args.release: generate_env_from_head(args) workflows = [] config = {} if args.config_file: config, new_workflows = parse_config_file(args.config_file, args.repos_dir) workflows.extend(new_workflows) # Add any paths to the python path extra_py_paths = [] for p in config.get("python_paths", []): # Assume that python_paths are in the format $REPO_OWNER/$REPO_NAME/path, # we need to ensure that the repo is checked out if it is different from # the current one, and if the repo is not kubeflow/testing (which is already # checked out). segments = p.split("/") if ((segments[0] != repo_owner or segments[1] != repo_name) and not p.startswith("kubeflow/testing")): logging.info("Need to clone %s/%s", segments[0], segments[1]) util.clone_repo( os.path.join(args.repos_dir, segments[0], segments[1]), segments[0], segments[1]) path = os.path.join(args.repos_dir, p) extra_py_paths.append(path) kf_test_path = os.path.join(args.repos_dir, "kubeflow/testing/py") if kf_test_path not in extra_py_paths: logging.info("Adding %s to extra python paths", kf_test_path) extra_py_paths.append(kf_test_path) logging.info("Extra python paths: %s", ":".join(extra_py_paths)) # Create an initial version of the file with no urls create_started_file(args.bucket, {}) util.maybe_activate_service_account() util.configure_kubectl(args.project, args.zone, args.cluster) util.load_kube_config() workflow_names = [] ui_urls = {} for w in workflows: # pylint: disable=too-many-nested-blocks # Create the name for the workflow # We truncate sha numbers to prevent the workflow name from being too large. # Workflow name should not be more than 63 characters because its used # as a label on the pods. workflow_name = os.getenv("JOB_NAME") + "-" + w.name # Skip this workflow if it is scoped to a different job type. if w.job_types and not job_type in w.job_types: logging.info( "Skipping workflow %s because job type %s is not one of " "%s.", w.name, job_type, w.job_types) continue # If we are scoping this workflow to specific directories, check if any files # modified match the specified regex patterns. dir_modified = False if w.include_dirs: for f in changed_files: for d in w.include_dirs: if fnmatch.fnmatch(f, d): dir_modified = True logging.info( "Triggering workflow %s because %s in dir %s is modified.", w.name, f, d) break if dir_modified: break # Only consider modified files when the job is pre or post submit, and if # the include_dirs stanza is defined. if job_type != "periodic" and w.include_dirs and not dir_modified: logging.info( "Skipping workflow %s because no code modified in %s.", w.name, w.include_dirs) continue if job_type == "presubmit": # When not running under prow we might not set all environment variables if os.getenv("PULL_NUMBER"): workflow_name += "-{0}".format(os.getenv("PULL_NUMBER")) if os.getenv("PULL_PULL_SHA"): workflow_name += "-{0}".format(os.getenv("PULL_PULL_SHA")[0:7]) elif job_type == "postsubmit": if os.getenv("PULL_BASE_SHA"): workflow_name += "-{0}".format(os.getenv("PULL_BASE_SHA")[0:7]) # Append the last 4 digits of the build number if os.getenv("BUILD_NUMBER"): workflow_name += "-{0}".format(os.getenv("BUILD_NUMBER")[-4:]) salt = uuid.uuid4().hex[0:4] # Add some salt. This is mostly a convenience for the case where you # are submitting jobs manually for testing/debugging. Since the prow should # vend unique build numbers for each job. workflow_name += "-{0}".format(salt) workflow_names.append(workflow_name) # check if ks workflow and run if w.app_dir: ks_cmd = ks_util.get_ksonnet_cmd(w.app_dir) # Print ksonnet version util.run([ks_cmd, "version"]) # Create a new environment for this run env = workflow_name util.run([ ks_cmd, "env", "add", env, "--namespace=" + get_namespace(args) ], cwd=w.app_dir) util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, "name", workflow_name ], cwd=w.app_dir) # Set the prow environment variables. prow_env = [] names = [ "JOB_NAME", "JOB_TYPE", "BUILD_ID", "BUILD_NUMBER", "PULL_BASE_SHA", "PULL_NUMBER", "PULL_PULL_SHA", "REPO_OWNER", "REPO_NAME" ] names.sort() for v in names: if not os.getenv(v): continue prow_env.append("{0}={1}".format(v, os.getenv(v))) util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, "prow_env", ",".join(prow_env) ], cwd=w.app_dir) util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, "namespace", get_namespace(args) ], cwd=w.app_dir) util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, "bucket", args.bucket ], cwd=w.app_dir) if args.release: util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, "versionTag", os.getenv("VERSION_TAG") ], cwd=w.app_dir) # Set any extra params. We do this in alphabetical order to make it easier to verify in # the unittest. param_names = w.params.keys() param_names.sort() for k in param_names: util.run([ ks_cmd, "param", "set", "--env=" + env, w.component, k, "{0}".format(w.params[k]) ], cwd=w.app_dir) # For debugging print out the manifest util.run([ks_cmd, "show", env, "-c", w.component], cwd=w.app_dir) util.run([ks_cmd, "apply", env, "-c", w.component], cwd=w.app_dir) ui_url = ( "http://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/{0}" "?tab=workflow".format(workflow_name)) ui_urls[workflow_name] = ui_url logging.info("URL for workflow: %s", ui_url) else: w.kwargs["name"] = workflow_name w.kwargs["namespace"] = get_namespace(args) if TEST_TARGET_ARG_NAME not in w.kwargs: w.kwargs[TEST_TARGET_ARG_NAME] = w.name logging.info( "Workflow %s doesn't set arg %s; defaulting to %s", w.name, TEST_TARGET_ARG_NAME, w.kwargs[TEST_TARGET_ARG_NAME]) # TODO(https://github.com/kubeflow/testing/issues/467): We shell out # to e2e_tool in order to dumpy the Argo workflow to a file which then # reimport. We do this because importing the py_func module appears # to break when we have to dynamically adjust sys.path to insert # new paths. Setting PYTHONPATH before launching python however appears # to work which is why we shell out to e2e_tool. command = [ "python", "-m", "kubeflow.testing.e2e_tool", "show", w.py_func ] for k, v in w.kwargs.items(): # The fire module turns underscores in parameter names into hyphens # so we convert underscores in parameter names to hyphens command.append("--{0}={1}".format(k.replace("_", "-"), v)) with tempfile.NamedTemporaryFile(delete=False) as hf: workflow_file = hf.name command.append("--output=" + hf.name) env = os.environ.copy() env["PYTHONPATH"] = ":".join(extra_py_paths) util.run(command, env=env) with open(workflow_file) as hf: wf_result = yaml.load(hf) group, version = wf_result['apiVersion'].split('/') k8s_co = k8s_client.CustomObjectsApi() workflow_name = wf_result["metadata"]["name"] py_func_result = k8s_co.create_namespaced_custom_object( group=group, version=version, namespace=wf_result["metadata"]["namespace"], plural='workflows', body=wf_result) logging.info("Created workflow:\n%s", yaml.safe_dump(py_func_result)) ui_url = ( "http://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/{0}" "?tab=workflow".format(workflow_name)) ui_urls[workflow_name] = ui_url logging.info("URL for workflow: %s", ui_url) # We delay creating started.json until we know the Argo workflow URLs create_started_file(args.bucket, ui_urls) workflow_success = False workflow_phase = {} workflow_status_yamls = {} results = [] try: results = argo_client.wait_for_workflows( get_namespace(args), workflow_names, timeout=datetime.timedelta(minutes=180), status_callback=argo_client.log_status) workflow_success = True except util.ExceptionWithWorkflowResults as e: # We explicitly log any exceptions so that they will be captured in the # build-log.txt that is uploaded to Gubernator. logging.exception("Exception occurred: %s", e) results = e.workflow_results raise finally: prow_artifacts_dir = prow_artifacts.get_gcs_dir(args.bucket) # Upload logs to GCS. No logs after this point will appear in the # file in gcs file_handler.flush() util.upload_file_to_gcs( file_handler.baseFilename, os.path.join(prow_artifacts_dir, "build-log.txt")) # Upload workflow status to GCS. for r in results: phase = r.get("status", {}).get("phase") name = r.get("metadata", {}).get("name") workflow_phase[name] = phase workflow_status_yamls[name] = yaml.safe_dump( r, default_flow_style=False) if phase != "Succeeded": workflow_success = False logging.info("Workflow %s/%s finished phase: %s", get_namespace(args), name, phase) for wf_name, wf_status in workflow_status_yamls.items(): util.upload_to_gcs( wf_status, os.path.join(prow_artifacts_dir, '{}.yaml'.format(wf_name))) all_tests_success = prow_artifacts.finalize_prow_job( args.bucket, workflow_success, workflow_phase, ui_urls) return all_tests_success
def run(args, file_handler): # pylint: disable=too-many-statements,too-many-branches # Check https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md # for a description of the injected environment variables. job_type = os.getenv("JOB_TYPE") repo_owner = os.getenv("REPO_OWNER") repo_name = os.getenv("REPO_NAME") base_branch_name = os.getenv("PULL_BASE_REF") pull_base_sha = os.getenv("PULL_BASE_SHA") # For presubmit/postsubmit jobs, find the list of files changed by the PR. diff_command = [] if job_type == "presubmit": # We need to get a common ancestor for the PR and the base branch cloned_repo_dir = os.path.join(args.repos_dir, repo_owner, repo_name) _ = util.run(["git", "fetch", "origin", base_branch_name + ":refs/remotes/origin/" + base_branch_name], cwd=cloned_repo_dir) diff_command = ["git", "diff", "--name-only"] diff_branch = "remotes/origin/{}".format(base_branch_name) try: common_ancestor = util.run(["git", "merge-base", "HEAD", diff_branch], cwd=cloned_repo_dir) diff_command.append(common_ancestor) except subprocess.CalledProcessError as e: logging.warning("git merge-base failed; see " "https://github.com/kubeflow/kubeflow/issues/3523. Diff " "will be computed against the current master and " "therefore files not changed in the PR might be " "considered when determining which tests to trigger") diff_command.append(diff_branch) elif job_type == "postsubmit": # See: https://git-scm.com/docs/git-diff # This syntax compares the commit before pull_base_sha with the commit # at pull_base_sha diff_command = ["git", "diff", "--name-only", pull_base_sha + "^", pull_base_sha] changed_files = [] if job_type in ("presubmit", "postsubmit"): changed_files = util.run(diff_command, cwd=os.path.join(args.repos_dir, repo_owner, repo_name)).splitlines() for f in changed_files: logging.info("File %s is modified.", f) if args.release: generate_env_from_head(args) workflows = [] if args.config_file: workflows.extend(parse_config_file(args.config_file, args.repos_dir)) # Create an initial version of the file with no urls create_started_file(args.bucket, {}) util.maybe_activate_service_account() util.configure_kubectl(args.project, args.zone, args.cluster) util.load_kube_config() workflow_names = [] ui_urls = {} for w in workflows: # Create the name for the workflow # We truncate sha numbers to prevent the workflow name from being too large. # Workflow name should not be more than 63 characters because its used # as a label on the pods. workflow_name = os.getenv("JOB_NAME") + "-" + w.name ks_cmd = ks_util.get_ksonnet_cmd(w.app_dir) # Print ksonnet version util.run([ks_cmd, "version"]) # Skip this workflow if it is scoped to a different job type. if w.job_types and not job_type in w.job_types: logging.info("Skipping workflow %s because job type %s is not one of " "%s.", w.name, job_type, w.job_types) continue # If we are scoping this workflow to specific directories, check if any files # modified match the specified regex patterns. dir_modified = False if w.include_dirs: for f in changed_files: for d in w.include_dirs: if fnmatch.fnmatch(f, d): dir_modified = True logging.info("Triggering workflow %s because %s in dir %s is modified.", w.name, f, d) break if dir_modified: break # Only consider modified files when the job is pre or post submit, and if # the include_dirs stanza is defined. if job_type != "periodic" and w.include_dirs and not dir_modified: logging.info("Skipping workflow %s because no code modified in %s.", w.name, w.include_dirs) continue if job_type == "presubmit": workflow_name += "-{0}".format(os.getenv("PULL_NUMBER")) workflow_name += "-{0}".format(os.getenv("PULL_PULL_SHA")[0:7]) elif job_type == "postsubmit": workflow_name += "-{0}".format(os.getenv("PULL_BASE_SHA")[0:7]) # Append the last 4 digits of the build number workflow_name += "-{0}".format(os.getenv("BUILD_NUMBER")[-4:]) salt = uuid.uuid4().hex[0:4] # Add some salt. This is mostly a convenience for the case where you # are submitting jobs manually for testing/debugging. Since the prow should # vend unique build numbers for each job. workflow_name += "-{0}".format(salt) workflow_names.append(workflow_name) # Create a new environment for this run env = workflow_name util.run([ks_cmd, "env", "add", env, "--namespace=" + get_namespace(args)], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "name", workflow_name], cwd=w.app_dir) # Set the prow environment variables. prow_env = [] names = ["JOB_NAME", "JOB_TYPE", "BUILD_ID", "BUILD_NUMBER", "PULL_BASE_SHA", "PULL_NUMBER", "PULL_PULL_SHA", "REPO_OWNER", "REPO_NAME"] names.sort() for v in names: if not os.getenv(v): continue prow_env.append("{0}={1}".format(v, os.getenv(v))) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "prow_env", ",".join(prow_env)], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "namespace", get_namespace(args)], cwd=w.app_dir) util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "bucket", args.bucket], cwd=w.app_dir) if args.release: util.run([ks_cmd, "param", "set", "--env=" + env, w.component, "versionTag", os.getenv("VERSION_TAG")], cwd=w.app_dir) # Set any extra params. We do this in alphabetical order to make it easier to verify in # the unittest. param_names = w.params.keys() param_names.sort() for k in param_names: util.run([ks_cmd, "param", "set", "--env=" + env, w.component, k, "{0}".format(w.params[k])], cwd=w.app_dir) # For debugging print out the manifest util.run([ks_cmd, "show", env, "-c", w.component], cwd=w.app_dir) util.run([ks_cmd, "apply", env, "-c", w.component], cwd=w.app_dir) ui_url = ("http://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/{0}" "?tab=workflow".format(workflow_name)) ui_urls[workflow_name] = ui_url logging.info("URL for workflow: %s", ui_url) # We delay creating started.json until we know the Argo workflow URLs create_started_file(args.bucket, ui_urls) workflow_success = False workflow_phase = {} workflow_status_yamls = {} results = [] try: results = argo_client.wait_for_workflows( get_namespace(args), workflow_names, timeout=datetime.timedelta(minutes=180), status_callback=argo_client.log_status ) workflow_success = True except util.ExceptionWithWorkflowResults as e: # We explicitly log any exceptions so that they will be captured in the # build-log.txt that is uploaded to Gubernator. logging.exception("Exception occurred: %s", e) results = e.workflow_results raise finally: prow_artifacts_dir = prow_artifacts.get_gcs_dir(args.bucket) # Upload logs to GCS. No logs after this point will appear in the # file in gcs file_handler.flush() util.upload_file_to_gcs( file_handler.baseFilename, os.path.join(prow_artifacts_dir, "build-log.txt")) # Upload workflow status to GCS. for r in results: phase = r.get("status", {}).get("phase") name = r.get("metadata", {}).get("name") workflow_phase[name] = phase workflow_status_yamls[name] = yaml.safe_dump(r, default_flow_style=False) if phase != "Succeeded": workflow_success = False logging.info("Workflow %s/%s finished phase: %s", get_namespace(args), name, phase) for wf_name, wf_status in workflow_status_yamls.items(): util.upload_to_gcs( wf_status, os.path.join(prow_artifacts_dir, '{}.yaml'.format(wf_name))) all_tests_success = prow_artifacts.finalize_prow_job( args.bucket, workflow_success, workflow_phase, ui_urls) return all_tests_success
def run(args, file_handler): # pylint: disable=too-many-statements,too-many-branches # Print ksonnet version util.run(["ks", "version"]) if args.release: generate_env_from_head(args) workflows = [] if args.config_file: workflows.extend(parse_config_file(args.config_file, args.repos_dir)) if args.app_dir and args.component: # TODO(jlewi): We can get rid of this branch once all repos are using a prow_config.xml file. workflows.append( WorkflowComponent("legacy", args.app_dir, args.component, {})) create_started_file(args.bucket) util.maybe_activate_service_account() util.configure_kubectl(args.project, args.zone, args.cluster) util.load_kube_config() api_client = k8s_client.ApiClient() workflow_names = [] ui_urls = {} for w in workflows: # Create the name for the workflow # We truncate sha numbers to prevent the workflow name from being too large. # Workflow name should not be more than 63 characters because its used # as a label on the pods. workflow_name = os.getenv("JOB_NAME") + "-" + w.name job_type = os.getenv("JOB_TYPE") if job_type == "presubmit": workflow_name += "-{0}".format(os.getenv("PULL_NUMBER")) workflow_name += "-{0}".format(os.getenv("PULL_PULL_SHA")[0:7]) elif job_type == "postsubmit": workflow_name += "-{0}".format(os.getenv("PULL_BASE_SHA")[0:7]) workflow_name += "-{0}".format(os.getenv("BUILD_NUMBER")) salt = uuid.uuid4().hex[0:4] # Add some salt. This is mostly a convenience for the case where you # are submitting jobs manually for testing/debugging. Since the prow should # vend unique build numbers for each job. workflow_name += "-{0}".format(salt) workflow_names.append(workflow_name) # Create a new environment for this run env = workflow_name util.run(["ks", "env", "add", env], cwd=w.app_dir) util.run([ "ks", "param", "set", "--env=" + env, w.component, "name", workflow_name ], cwd=w.app_dir) # Set the prow environment variables. prow_env = [] names = [ "JOB_NAME", "JOB_TYPE", "BUILD_ID", "BUILD_NUMBER", "PULL_BASE_SHA", "PULL_NUMBER", "PULL_PULL_SHA", "REPO_OWNER", "REPO_NAME" ] names.sort() for v in names: if not os.getenv(v): continue prow_env.append("{0}={1}".format(v, os.getenv(v))) util.run([ "ks", "param", "set", "--env=" + env, w.component, "prow_env", ",".join(prow_env) ], cwd=w.app_dir) util.run([ "ks", "param", "set", "--env=" + env, w.component, "namespace", get_namespace(args) ], cwd=w.app_dir) util.run([ "ks", "param", "set", "--env=" + env, w.component, "bucket", args.bucket ], cwd=w.app_dir) if args.release: util.run([ "ks", "param", "set", "--env=" + env, w.component, "versionTag", os.getenv("VERSION_TAG") ], cwd=w.app_dir) # Set any extra params. We do this in alphabetical order to make it easier to verify in # the unittest. param_names = w.params.keys() param_names.sort() for k in param_names: util.run([ "ks", "param", "set", "--env=" + env, w.component, k, "{0}".format(w.params[k]) ], cwd=w.app_dir) # For debugging print out the manifest util.run(["ks", "show", env, "-c", w.component], cwd=w.app_dir) util.run(["ks", "apply", env, "-c", w.component], cwd=w.app_dir) ui_url = ( "http://testing-argo.kubeflow.org/workflows/kubeflow-test-infra/{0}" "?tab=workflow".format(workflow_name)) ui_urls[workflow_name] = ui_url logging.info("URL for workflow: %s", ui_url) success = True workflow_phase = {} try: results = argo_client.wait_for_workflows( api_client, get_namespace(args), workflow_names, status_callback=argo_client.log_status) for r in results: phase = r.get("status", {}).get("phase") name = r.get("metadata", {}).get("name") workflow_phase[name] = phase if phase != "Succeeded": success = False logging.info("Workflow %s/%s finished phase: %s", get_namespace(args), name, phase) except util.TimeoutError: success = False logging.error("Time out waiting for Workflows %s to finish", ",".join(workflow_names)) except Exception as e: # We explicitly log any exceptions so that they will be captured in the # build-log.txt that is uploaded to Gubernator. logging.error("Exception occurred: %s", e) raise finally: success = prow_artifacts.finalize_prow_job(args.bucket, success, workflow_phase, ui_urls) # Upload logs to GCS. No logs after this point will appear in the # file in gcs file_handler.flush() util.upload_file_to_gcs( file_handler.baseFilename, os.path.join(prow_artifacts.get_gcs_dir(args.bucket), "build-log.txt")) return success
def run(args, file_handler): create_started_file(args.bucket) util.maybe_activate_service_account() util.configure_kubectl(args.project, args.zone, args.cluster) util.load_kube_config() # Create the name for the workflow # We truncate sha numbers to prevent the workflow name from being too large. # Workflow name should not be more than 63 characters because its used # as a label on the pods. workflow_name = os.getenv("JOB_NAME") job_type = os.getenv("JOB_TYPE") if job_type == "presubmit": workflow_name += "-{0}".format(os.getenv("PULL_NUMBER")) workflow_name += "-{0}".format(os.getenv("PULL_PULL_SHA")[0:7]) elif job_type == "postsubmit": workflow_name += "-{0}".format(os.getenv("PULL_BASE_SHA")[0:7]) workflow_name += "-{0}".format(os.getenv("BUILD_NUMBER")) salt = uuid.uuid4().hex[0:4] # Add some salt. This is mostly a convenience for the case where you # are submitting jobs manually for testing/debugging. Since the prow should # vend unique build numbers for each job. workflow_name += "-{0}".format(salt) # Create a new environment for this run env = workflow_name util.run(["ks", "env", "add", env], cwd=args.app_dir) util.run([ "ks", "param", "set", "--env=" + env, args.component, "name", workflow_name ], cwd=args.app_dir) util.load_kube_config() api_client = k8s_client.ApiClient() # Set the prow environment variables. prow_env = [] names = [ "JOB_NAME", "JOB_TYPE", "BUILD_ID", "BUILD_NUMBER", "PULL_BASE_SHA", "PULL_NUMBER", "PULL_PULL_SHA", "REPO_OWNER", "REPO_NAME" ] names.sort() for v in names: if not os.getenv(v): continue prow_env.append("{0}={1}".format(v, os.getenv(v))) util.run([ "ks", "param", "set", "--env=" + env, args.component, "prow_env", ",".join(prow_env) ], cwd=args.app_dir) util.run([ "ks", "param", "set", "--env=" + env, args.component, "namespace", NAMESPACE ], cwd=args.app_dir) util.run([ "ks", "param", "set", "--env=" + env, args.component, "bucket", args.bucket ], cwd=args.app_dir) # For debugging print out the manifest util.run(["ks", "show", env, "-c", args.component], cwd=args.app_dir) util.run(["ks", "apply", env, "-c", args.component], cwd=args.app_dir) ui_url = ( "http://testing-argo.kubeflow.io/timeline/kubeflow-test-infra/{0}" ";tab=workflow".format(workflow_name)) logging.info("URL for workflow: %s", ui_url) success = False try: results = argo_client.wait_for_workflow( api_client, NAMESPACE, workflow_name, status_callback=argo_client.log_status) if results["status"]["phase"] == "Succeeded": success = True logging.info("Workflow %s/%s finished phase: %s", NAMESPACE, workflow_name, results["status"]["phase"]) except util.TimeoutError: success = False logging.error("Time out waiting for Workflow %s/%s to finish", NAMESPACE, workflow_name) finally: create_finished_file(args.bucket, success) # Upload logs to GCS. No logs after this point will appear in the # file in gcs file_handler.flush() upload_file_to_gcs( file_handler.baseFilename, os.path.join(prow_artifacts.get_gcs_dir(args.bucket), "build-log.txt")) return success
def create_finished_file(bucket, success): """Create the started file in gcs for gubernator.""" contents = prow_artifacts.create_finished(success) target = os.path.join(prow_artifacts.get_gcs_dir(bucket), "finished.json") upload_to_gcs(contents, target)