def test_write_xml(self):
with tempfile.NamedTemporaryFile(delete=False) as hf:
pass
success = test_util.TestCase()
success.class_name = "some_test"
success.name = "first"
success.time = 10
failure = test_util.TestCase()
failure.class_name = "some_test"
failure.name = "first"
failure.time = 10
failure.failure = "failed for some reason."
test_util.create_junit_xml_file([success, failure], hf.name)
with open(hf.name) as hf:
output = hf.read()
print(output)
expected = ("""<testsuite failures="1" tests="2" time="20">"""
"""<testcase classname="some_test" name="first" time="10" />"""
"""<testcase classname="some_test" name="first" """
"""time="10"><failure>failed for some reason.</failure>"""
"""</testcase></testsuite>""")
self.assertEqual(expected, output)
def run_test(test_case, test_func, args): # pylint: disable=too-many-branches,too-many-statements
"""Run a test."""
gcs_client = storage.Client(project=args.project)
project = args.project
cluster_name = args.cluster
zone = args.zone
# TODO(jlewi): When using GKE we should copy the .kube config and any other
# files to the test directory. We should then set the environment variable
# KUBECONFIG to point at that file. This should prevent us from having
# to rerun util.configure_kubectl on each step. Instead we could run it once
# as part of GKE cluster creation and store the config in the NFS directory.
# This would make the handling of credentials
# and KUBECONFIG more consistent between GKE and minikube and eventually
# this could be extended to other K8s deployments.
if cluster_name:
util.configure_kubectl(project, zone, cluster_name)
util.load_kube_config()
start = time.time()
try: # pylint: disable=too-many-nested-blocks
# We repeat the test multiple times.
# This ensures that if we delete the job we can create a new job with the
# same name.
num_trials = args.num_trials
logging.info("tfjob_version=%s", args.tfjob_version)
for trial in range(num_trials):
logging.info("Trial %s", trial)
test_func()
# TODO(jlewi):
# Here are some validation checks to run:
# 1. Check that all resources are garbage collected.
# TODO(jlewi): Add an option to add chaos and randomly kill various resources?
# TODO(jlewi): Are there other generic validation checks we should
# run.
except tf_operator_util.JobTimeoutError as e:
if e.job:
spec = "Job:\n" + json.dumps(e.job, indent=2)
else:
spec = "JobTimeoutError did not contain job"
test_case.failure = "Timeout waiting for job to finish: " + spec
logging.exception(test_case.failure)
except Exception as e: # pylint: disable-msg=broad-except
# TODO(jlewi): I'm observing flakes where the exception has message "status"
# in an effort to try to nail down this exception we print out more
# information about the exception.
logging.exception("There was a problem running the job; Exception %s", e)
# We want to catch all exceptions because we want the test as failed.
test_case.failure = ("Exception occured; type {0} message {1}".format(
e.__class__, e.message))
finally:
test_case.time = time.time() - start
if args.artifacts_path:
test_util.create_junit_xml_file(
[test_case],
args.artifacts_path + "/junit_" + test_func.__name__ + ".xml",
gcs_client)
def main(): # pylint: disable=too-many-locals
logging.getLogger().setLevel(logging.INFO) # pylint: disable=too-many-locals
# create the top-level parser
parser = argparse.ArgumentParser(description="jsonnet tests.")
parser.add_argument(
"--test_files_dirs",
default="",
type=str,
help=
"Comma separated directories where test jsonnet test files are stored")
parser.add_argument(
"--artifacts_dir",
default="",
type=str,
help="Directory to use for artifacts that should be preserved after "
"the test runs. Defaults to test_dir if not set.")
args = parser.parse_args()
if not args.test_files_dirs:
raise ValueError('--test_files_dirs needs to be set')
test_log = os.path.join(args.artifacts_dir, "logs", "test_jsonnet.log.txt")
if not os.path.exists(os.path.dirname(test_log)):
os.makedirs(os.path.dirname(test_log))
root_logger = logging.getLogger()
file_handler = logging.FileHandler(test_log)
root_logger.addHandler(file_handler)
# We need to explicitly set the formatter because it will not pick up
# the BasicConfig.
formatter = logging.Formatter(
fmt=("%(levelname)s|%(asctime)s"
"|%(pathname)s|%(lineno)d| %(message)s"),
datefmt="%Y-%m-%dT%H:%M:%S")
file_handler.setFormatter(formatter)
logging.info("Logging to %s", test_log)
t = test_util.TestCase()
t.class_name = "Kubeflow"
t.name = "test-jsonnet"
test_files_dirs = args.test_files_dirs.split(',')
start = time.time()
try:
run(test_files_dirs, t)
finally:
t.time = time.time() - start
junit_path = os.path.join(args.artifacts_dir,
"junit_kubeflow-test-jsonnet.xml")
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([t], junit_path)
def run_tests(args):
# Print out the pylint version because different versions can produce
# different results.
util.run(["pylint", "--version"])
# kubeflow_testing is imported as a submodule so we should exclude it
# TODO(jlewi): Perhaps we should get a list of submodules and exclude
# them automatically?
dir_excludes = ["vendor"]
includes = ["*_test.py"]
test_cases = []
num_failed = 0
for root, dirs, files in os.walk(args.src_dir, topdown=True):
# excludes can be done with fnmatch.filter and complementary set,
# but it's more annoying to read.
dirs[:] = [d for d in dirs if d not in dir_excludes]
for pat in includes:
for f in fnmatch.filter(files, pat):
full_path = os.path.join(root, f)
test_case = test_util.TestCase()
test_case.class_name = "pytest"
test_case.name = full_path[len(args.src_dir):]
start_time = time.time()
test_cases.append(test_case)
try:
util.run(["python", full_path], cwd=args.src_dir)
except subprocess.CalledProcessError:
test_case.failure = "{0} failed.".format(test_case.name)
num_failed += 1
finally:
test_case.time = time.time() - start_time
if num_failed:
logging.error("%s tests failed.", num_failed)
else:
logging.info("No lint issues.")
if not args.junit_path:
logging.info("No --junit_path.")
return
gcs_client = None
if args.junit_path.startswith("gs://"):
gcs_client = storage.Client(project=args.project)
test_util.create_junit_xml_file(test_cases, args.junit_path, gcs_client)
def run_test(test_case, test_func, args): # pylint: disable=too-many-branches,too-many-statements
"""Run a test."""
util.load_kube_config()
start = time.time()
try: # pylint: disable=too-many-nested-blocks
# We repeat the test multiple times.
# This ensures that if we delete the job we can create a new job with the
# same name.
num_trials = args.num_trials
logging.info("tfjob_version=%s", args.tfjob_version)
for trial in range(num_trials):
logging.info("Trial %s", trial)
test_func()
# TODO(jlewi):
# Here are some validation checks to run:
# 1. Check that all resources are garbage collected.
# TODO(jlewi): Add an option to add chaos and randomly kill various resources?
# TODO(jlewi): Are there other generic validation checks we should
# run.
except tf_operator_util.JobTimeoutError as e:
if e.job:
spec = "Job:\n" + json.dumps(e.job, indent=2)
else:
spec = "JobTimeoutError did not contain job"
test_case.failure = "Timeout waiting for job to finish: " + spec
logging.exception(test_case.failure)
except Exception as e: # pylint: disable-msg=broad-except
# TODO(jlewi): I'm observing flakes where the exception has message "status"
# in an effort to try to nail down this exception we print out more
# information about the exception.
logging.exception("There was a problem running the job; Exception %s",
e)
# We want to catch all exceptions because we want the test as failed.
test_case.failure = ("Exception occured; type {0} message {1}".format(
e.__class__, e.message))
finally:
test_case.time = time.time() - start
if args.artifacts_path:
test_util.create_junit_xml_file([test_case],
args.artifacts_path + "/junit_" +
test_func.__name__ + ".xml")
def wrap_test(args):
"""Run the tests given by args.func and output artifacts as necessary.
"""
test_name = determine_test_name(args)
test_case = test_util.TestCase()
test_case.class_name = "KubeFlow"
test_case.name = "deploy-kubeflow-" + test_name
try:
def run():
args.func(args)
test_util.wrap_test(run, test_case)
finally:
junit_path = os.path.join(
args.artifacts_dir, "junit_kubeflow-deploy-{0}.xml".format(test_name))
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([test_case], junit_path)
def wrap_test(args):
"""Run the tests given by args.func and output artifacts as necessary.
"""
test_name = determine_test_name(args)
test_case = test_util.TestCase()
test_case.class_name = "KubeFlow"
test_case.name = args.workflow_name + "-" + test_name
try:
def run():
args.func(args)
test_util.wrap_test(run, test_case)
finally:
# Test grid has problems with underscores in the name.
# https://github.com/kubeflow/kubeflow/issues/631
# TestGrid currently uses the regex junit_(^_)*.xml so we only
# want one underscore after junit.
junit_name = test_case.name.replace("_", "-")
junit_path = os.path.join(args.artifacts_dir,
"junit_{0}.xml".format(junit_name))
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([test_case], junit_path)
def main():
parser = argparse.ArgumentParser('Label an image using Inception')
parser.add_argument('-p',
'--port',
type=int,
default=9000,
help='Port at which Inception model is being served')
parser.add_argument("--namespace",
required=True,
type=str,
help=("The namespace to use."))
parser.add_argument("--service_name",
required=True,
type=str,
help=("The TF serving service to use."))
parser.add_argument(
"--artifacts_dir",
default="",
type=str,
help="Directory to use for artifacts that should be preserved after "
"the test runs. Defaults to test_dir if not set.")
parser.add_argument("--input_path",
required=True,
type=str,
help=("The input file to use."))
parser.add_argument("--result_path",
type=str,
help=("The expected result."))
parser.add_argument("--workflow_name",
default="tfserving",
type=str,
help="The name of the workflow.")
args = parser.parse_args()
t = test_util.TestCase()
t.class_name = "Kubeflow"
t.name = args.workflow_name + "-" + args.service_name
start = time.time()
util.load_kube_config(persist_config=False)
api_client = k8s_client.ApiClient()
core_api = k8s_client.CoreV1Api(api_client)
try:
with open(args.input_path) as f:
instances = json.loads(f.read())
service = core_api.read_namespaced_service(args.service_name,
args.namespace)
service_ip = service.spec.cluster_ip
model_urls = [
"http://" + service_ip +
":8500/v1/models/mnist:predict", # tf serving's http server
]
for model_url in model_urls:
logging.info("Try predicting with endpoint {}".format(model_url))
num_try = 1
result = None
while True:
try:
result = requests.post(model_url, json=instances)
assert (result.status_code == 200)
except Exception as e:
num_try += 1
if num_try > 10:
raise
logging.info(
'prediction failed: {}. Retrying...'.format(e))
time.sleep(5)
else:
break
logging.info('Got result: {}'.format(result.text))
if args.result_path:
with open(args.result_path) as f:
expected_result = json.loads(f.read())
logging.info('Expected result: {}'.format(expected_result))
assert (almost_equal(expected_result,
json.loads(result.text)))
except Exception as e:
t.failure = "Test failed; " + e.message
raise
finally:
t.time = time.time() - start
junit_path = os.path.join(
args.artifacts_dir,
"junit_kubeflow-tf-serving-image-{}.xml".format(args.service_name))
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([t], junit_path)
# Pause to collect Stackdriver logs.
time.sleep(60)
def main():
parser = argparse.ArgumentParser('Label an image using Inception')
parser.add_argument('-p',
'--port',
type=int,
default=9000,
help='Port at which Inception model is being served')
parser.add_argument("--namespace",
required=True,
type=str,
help=("The namespace to use."))
parser.add_argument("--service_name",
required=True,
type=str,
help=("The TF serving service to use."))
parser.add_argument(
"--artifacts_dir",
default="",
type=str,
help="Directory to use for artifacts that should be preserved after "
"the test runs. Defaults to test_dir if not set.")
parser.add_argument("--image_path",
required=True,
type=str,
help=("The image to use."))
parser.add_argument("--result_path",
type=str,
help=("The expected result."))
args = parser.parse_args()
t = test_util.TestCase()
t.class_name = "Kubeflow"
t.name = "tf-serving-image"
start = time.time()
try:
server = "{}.{}.svc.cluster.local".format(args.service_name,
args.namespace)
channel = implementations.insecure_channel(server, args.port)
stub = prediction_service_pb2.beta_create_PredictionService_stub(
channel)
with tf.gfile.Open(args.image_path) as img:
raw_image = (img.read())
# Send request
# See prediction_service.proto for gRPC request/response details.
request = predict_pb2.PredictRequest()
request.model_spec.name = 'inception'
request.model_spec.signature_name = 'predict_images'
request.inputs['images'].CopyFrom(
tf.make_tensor_proto(raw_image, shape=[
1,
]))
num_try = 1
result = None
while True:
try:
result = str(stub.Predict(request, 10.0)) # 10 secs timeout
except Exception as e:
num_try += 1
if num_try > 3:
raise
logging.info('prediction failed: {}. Retrying...'.format(e))
time.sleep(5)
else:
break
logging.info('Got result: {}'.format(result))
if args.result_path:
with open(args.result_path) as f:
expected_result = f.read()
logging.info('Expected result: {}'.format(expected_result))
assert (expected_result == result)
except Exception as e:
t.failure = "Test failed; " + e.message
finally:
t.time = time.time() - start
junit_path = os.path.join(args.artifacts_dir,
"junit_kubeflow-tf-serving-image.xml")
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([t], junit_path)
def setup(args):
"""Test deploying Kubeflow."""
if args.cluster:
project = args.project
cluster_name = args.cluster
zone = args.zone
logging.info("Using cluster: %s in project: %s in zone: %s",
cluster_name, project, zone)
# Print out config to help debug issues with accounts and
# credentials.
util.run(["gcloud", "config", "list"])
util.configure_kubectl(project, zone, cluster_name)
util.load_kube_config()
else:
# TODO(jlewi): This is sufficient for API access but it doesn't create
# a kubeconfig file which ksonnet needs for ks init.
logging.info("Running inside cluster.")
incluster_config.load_incluster_config()
# Create an API client object to talk to the K8s master.
api_client = k8s_client.ApiClient()
now = datetime.datetime.now()
run_label = "e2e-" + now.strftime("%m%d-%H%M-") + uuid.uuid4().hex[0:4]
if not os.path.exists(args.test_dir):
os.makedirs(args.test_dir)
logging.info("Using test directory: %s", args.test_dir)
namespace_name = run_label
def run():
namespace = _setup_test(api_client, namespace_name)
logging.info("Using namespace: %s", namespace)
# Set a GITHUB_TOKEN so that we don't rate limited by GitHub;
# see: https://github.com/ksonnet/ksonnet/issues/233
os.environ["GITHUB_TOKEN"] = args.github_token
# Initialize a ksonnet app.
app_name = "kubeflow-test"
util.run(["ks", "init", app_name,], cwd=args.test_dir)
app_dir = os.path.join(args.test_dir, app_name)
kubeflow_registry = "github.com/kubeflow/kubeflow/tree/master/kubeflow"
util.run(["ks", "registry", "add", "kubeflow", kubeflow_registry], cwd=app_dir)
# Install required packages
packages = ["kubeflow/core", "kubeflow/tf-serving", "kubeflow/tf-job"]
for p in packages:
util.run(["ks", "pkg", "install", p], cwd=app_dir)
# Delete the vendor directory and replace with a symlink to the src
# so that we use the code at the desired commit.
target_dir = os.path.join(app_dir, "vendor", "kubeflow")
logging.info("Deleting %s", target_dir)
shutil.rmtree(target_dir)
REPO_ORG = "kubeflow"
REPO_NAME = "kubeflow"
REGISTRY_PATH = "kubeflow"
source = os.path.join(args.test_dir, "src", REPO_ORG, REPO_NAME,
REGISTRY_PATH)
logging.info("Creating link %s -> %s", target_dir, source)
os.symlink(source, target_dir)
# Deploy Kubeflow
util.run(["ks", "generate", "core", "kubeflow-core", "--name=kubeflow-core",
"--namespace=" + namespace.metadata.name], cwd=app_dir)
# TODO(jlewi): For reasons I don't understand even though we ran
# configure_kubectl above, if we don't rerun it we get rbac errors
# when we do ks apply; I think because we aren't using the proper service
# account. This might have something to do with the way ksonnet gets
# its credentials; maybe we need to configure credentials after calling
# ks init?
if args.cluster:
util.configure_kubectl(args.project, args.zone, args.cluster)
apply_command = ["ks", "apply", "default", "-c", "kubeflow-core",]
util.run(apply_command, cwd=app_dir)
# Verify that the TfJob operator is actually deployed.
tf_job_deployment_name = "tf-job-operator"
logging.info("Verifying TfJob controller started.")
util.wait_for_deployment(api_client, namespace.metadata.name,
tf_job_deployment_name)
# Verify that JupyterHub is actually deployed.
jupyter_name = "tf-hub"
logging.info("Verifying TfHub started.")
util.wait_for_statefulset(api_client, namespace.metadata.name, jupyter_name)
main_case = test_util.TestCase()
main_case.class_name = "KubeFlow"
main_case.name = "deploy-kubeflow"
try:
test_util.wrap_test(run, main_case)
finally:
# Delete the namespace
logging.info("Deleting namespace %s", namespace_name)
# We report teardown as a separate test case because this will help
# us track down issues with garbage collecting namespaces.
teardown = test_util.TestCase(main_case.class_name, "teardown")
def run_teardown():
core_api = k8s_client.CoreV1Api(api_client)
core_api.delete_namespace(namespace_name, {})
try:
test_util.wrap_test(run_teardown, teardown)
except Exception as e: # pylint: disable-msg=broad-except
logging.error("There was a problem deleting namespace: %s; %s",
namespace_name, e.message)
junit_path = os.path.join(args.artifacts_dir, "junit_kubeflow-deploy.xml")
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([main_case, teardown], junit_path)
def run_lint(args):
start_time = time.time()
# Print out the pylint version because different versions can produce
# different results.
util.run(["pylint", "--version"])
# kubeflow_testing is imported as a submodule so we should exclude it
# TODO(jlewi): We should make this an argument.
dir_excludes = [
"dashboard/frontend/node_modules",
"kubeflow_testing",
"vendor",
]
full_dir_excludes = [
os.path.join(os.path.abspath(args.src_dir), f) for f in dir_excludes
]
# TODO(jlewi): Use pathlib once we switch to python3.
includes = ["*.py"]
failed_files = []
rc_file = os.path.join(args.src_dir, ".pylintrc")
for root, dirs, files in os.walk(os.path.abspath(args.src_dir),
topdown=True):
# excludes can be done with fnmatch.filter and complementary set,
# but it's more annoying to read.
if should_exclude(root, full_dir_excludes):
continue
dirs[:] = [d for d in dirs]
for pat in includes:
for f in fnmatch.filter(files, pat):
full_path = os.path.join(root, f)
try:
util.run(["pylint", "--rcfile=" + rc_file, full_path],
cwd=args.src_dir)
except subprocess.CalledProcessError:
failed_files.append(full_path[len(args.src_dir):])
if failed_files:
failed_files.sort()
logging.error("%s files had lint errors:\n%s", len(failed_files),
"\n".join(failed_files))
else:
logging.info("No lint issues.")
if not args.junit_path:
logging.info("No --junit_path.")
return
test_case = test_util.TestCase()
test_case.class_name = "pylint"
test_case.name = "pylint"
test_case.time = time.time() - start_time
if failed_files:
test_case.failure = "Files with lint issues: {0}".format(
", ".join(failed_files))
gcs_client = None
if args.junit_path.startswith("gs://"):
gcs_client = storage.Client(project=args.project)
test_util.create_junit_xml_file([test_case], args.junit_path, gcs_client)
def setup(args):
"""Test deploying Kubeflow."""
if args.cluster:
project = args.project
cluster_name = args.cluster
zone = args.zone
logging.info("Using cluster: %s in project: %s in zone: %s",
cluster_name, project, zone)
# Print out config to help debug issues with accounts and
# credentials.
util.run(["gcloud", "config", "list"])
util.configure_kubectl(project, zone, cluster_name)
util.load_kube_config()
else:
# TODO(jlewi): This is sufficient for API access but it doesn't create
# a kubeconfig file which ksonnet needs for ks init.
logging.info("Running inside cluster.")
incluster_config.load_incluster_config()
# Create an API client object to talk to the K8s master.
api_client = k8s_client.ApiClient()
now = datetime.datetime.now()
run_label = "e2e-" + now.strftime("%m%d-%H%M-") + uuid.uuid4().hex[0:4]
if not os.path.exists(args.test_dir):
os.makedirs(args.test_dir)
logging.info("Using test directory: %s", args.test_dir)
namespace_name = run_label
def run():
namespace = _setup_test(api_client, namespace_name)
logging.info("Using namespace: %s", namespace)
# Set a GITHUB_TOKEN so that we don't rate limited by GitHub;
# see: https://github.com/ksonnet/ksonnet/issues/233
os.environ["GITHUB_TOKEN"] = args.github_token
# Initialize a ksonnet app.
app_name = "kubeflow-test"
util.run(["ks", "init", app_name,], cwd=args.test_dir)
app_dir = os.path.join(args.test_dir, app_name)
kubeflow_registry = "github.com/kubeflow/kubeflow/tree/master/kubeflow"
util.run(["ks", "registry", "add", "kubeflow", kubeflow_registry], cwd=app_dir)
# Install required packages
packages = ["kubeflow/core", "kubeflow/tf-serving", "kubeflow/tf-job"]
for p in packages:
util.run(["ks", "pkg", "install", p], cwd=app_dir)
# Delete the vendor directory and replace with a symlink to the src
# so that we use the code at the desired commit.
target_dir = os.path.join(app_dir, "vendor", "kubeflow")
logging.info("Deleting %s", target_dir)
shutil.rmtree(target_dir)
REPO_ORG = "kubeflow"
REPO_NAME = "kubeflow"
REGISTRY_PATH = "kubeflow"
source = os.path.join(args.test_dir, "src", REPO_ORG, REPO_NAME,
REGISTRY_PATH)
logging.info("Creating link %s -> %s", target_dir, source)
os.symlink(source, target_dir)
# Deploy Kubeflow
util.run(["ks", "generate", "core", "kubeflow-core", "--name=kubeflow-core",
"--namespace=" + namespace.metadata.name], cwd=app_dir)
# TODO(jlewi): For reasons I don't understand even though we ran
# configure_kubectl above, if we don't rerun it we get rbac errors
# when we do ks apply; I think because we aren't using the proper service
# account. This might have something to do with the way ksonnet gets
# its credentials; maybe we need to configure credentials after calling
# ks init?
if args.cluster:
util.configure_kubectl(args.project, args.zone, args.cluster)
apply_command = ["ks", "apply", "default", "-c", "kubeflow-core",]
util.run(apply_command, cwd=app_dir)
# Verify that the TfJob operator is actually deployed.
tf_job_deployment_name = "tf-job-operator"
logging.info("Verifying TfJob controller started.")
util.wait_for_deployment(api_client, namespace.metadata.name,
tf_job_deployment_name)
# Verify that JupyterHub is actually deployed.
jupyter_name = "tf-hub"
logging.info("Verifying TfHub started.")
util.wait_for_statefulset(api_client, namespace.metadata.name, jupyter_name)
main_case = test_util.TestCase()
main_case.class_name = "KubeFlow"
main_case.name = "deploy-kubeflow"
try:
test_util.wrap_test(run, main_case)
finally:
# Delete the namespace
logging.info("Deleting namespace %s", namespace_name)
# We report teardown as a separate test case because this will help
# us track down issues with garbage collecting namespaces.
teardown = test_util.TestCase(main_case.class_name, "teardown")
def run_teardown():
core_api = k8s_client.CoreV1Api(api_client)
core_api.delete_namespace(namespace_name, {})
try:
test_util.wrap_test(run_teardown, teardown)
except Exception as e: # pylint: disable-msg=broad-except
logging.error("There was a problem deleting namespace: %s; %s",
namespace_name, e.message)
junit_path = os.path.join(args.artifacts_dir, "junit_kubeflow-deploy.xml")
logging.info("Writing test results to %s", junit_path)
test_util.create_junit_xml_file([main_case, teardown], junit_path)