def test_run_job(self): success_condition = "status.succeeded > 0" failure_condition = "status.failed > 3" manifest = """apiVersion: batch/v1 kind: Job metadata: generateName: rand-num- spec: template: spec: containers: - name: rand image: python:3.6 command: ["python random_num.py"] """ couler.run_job( manifest=manifest, success_condition=success_condition, failure_condition=failure_condition, step_name="test_run_job", ) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] t = proto_wf.templates[s.tmpl_name] self.assertEqual(s.resource_spec.manifest, manifest) self.assertEqual(s.resource_spec.success_condition, success_condition) self.assertEqual(s.resource_spec.failure_condition, failure_condition) self.assertEqual(len(t.outputs), 3) self.assertEqual(t.outputs[0].parameter.name, "job-name")
def test_when(self): def random_code(): import random res = "heads" if random.randint(0, 1) == 0 else "tails" print(res) def heads(): return couler.run_container( image="alpine:3.6", command=["sh", "-c", 'echo "it was heads"']) def tails(): return couler.run_container( image="alpine:3.6", command=["sh", "-c", 'echo "it was tails"']) result = couler.run_script(image="python:alpine3.6", source=random_code) couler.when(couler.equal(result, "heads"), lambda: heads()) couler.when(couler.equal(result, "tails"), lambda: tails()) proto_wf = get_default_proto_workflow() step_heads = proto_wf.steps[1].steps[0] # condition is like: "{{steps.test-when-550.outputs.result}} == heads" self.assertTrue(step_heads.when.startswith("{{steps.test-when-")) self.assertTrue(step_heads.when.endswith(".outputs.result}} == heads"))
def test_output_oss_artifact(self): # the content of local file would be uploaded to OSS output_artifact = couler.create_oss_artifact( path="/home/t1.txt", bucket="test-bucket/", accesskey_id="abcde", accesskey_secret="abc12345", key="osspath/t1", endpoint="xyz.com", ) couler.run_container( image="docker/whalesay:latest", args=["echo -n hello world > %s" % output_artifact.path], command=["bash", "-c"], output=output_artifact, ) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] t = proto_wf.templates[s.tmpl_name] self.assertEqual(s.container_spec.image, "docker/whalesay:latest") self.assertTrue(t.outputs[0].artifact.name.startswith("output-oss")) self.assertEqual(t.outputs[0].artifact.local_path, "/home/t1.txt") self.assertEqual(t.outputs[0].artifact.endpoint, "xyz.com") self.assertEqual(t.outputs[0].artifact.bucket, "test-bucket/") self.assertEqual(t.outputs[0].artifact.access_key.key, "accessKey") proto_sk = t.outputs[0].artifact.secret_key self.assertEqual(proto_sk.key, "secretKey") self.assertEqual(proto_sk.value, states._secrets[proto_sk.name].data[proto_sk.key])
def test_script_step(self): def echo(): print("echo") couler.run_script(image="docker/whalesay:latest", source=echo) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] self.assertEqual(s.script, '\nprint("echo")\n')
def test_exit_handler(self): def send_mail(): return couler.run_container(image="alpine:3.6", command=["echo", "send mail"]) couler.run_container(image="alpine:3.6", command=["exit", "1"]) couler.set_exit_handler(couler.WFStatus.Failed, send_mail) proto_wf = get_default_proto_workflow() self.assertEqual(len(proto_wf.exit_handler_steps), 1) s = proto_wf.exit_handler_steps[0] self.assertEqual(s.when, "{{workflow.status}} == Failed")
def __init__( self, namespace="default", config_file=None, context=None, client_configuration=None, persist_config=True, ): logging.basicConfig(level=logging.INFO) self.namespace = namespace self.go_impl = ( os.environ.get( _SUBMITTER_IMPL_ENV_VAR_KEY, _SubmitterImplTypes.PYTHON ) == _SubmitterImplTypes.GO ) if self.go_impl: from ctypes import c_char_p, cdll from couler.core.proto_repr import get_default_proto_workflow self.go_submitter = cdll.LoadLibrary("./submit.so") self.go_submitter.Submit.argtypes = [c_char_p, c_char_p, c_char_p] self.go_submitter.Submit.restype = c_char_p with tempfile.NamedTemporaryFile( dir="/tmp", delete=False, mode="wb" ) as tmp_file: self.proto_path = tmp_file.name proto_wf = get_default_proto_workflow() tmp_file.write(proto_wf.SerializeToString()) else: try: config.load_kube_config( config_file, context, client_configuration, persist_config ) logging.info( "Found local kubernetes config. " "Initialized with kube_config." ) except Exception: logging.info( "Cannot find local k8s config. " "Trying in-cluster config." ) config.load_incluster_config() logging.info("Initialized with in-cluster config.") self._custom_object_api_client = k8s_client.CustomObjectsApi() self._core_api_client = k8s_client.CoreV1Api()
def test_script_step(self): def echo(): print("echo") couler.run_script( image="docker/whalesay:latest", source=echo, resources={ "cpu": "2", "memory": "1Gi" }, ) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] self.assertFalse(s.HasField("resource_spec")) self.assertEqual(s.script, '\nprint("echo")\n') self.assertEqual(s.container_spec.resources["cpu"], "2")
def test_output_oss_artifact(self): # the content of local file would be uploaded to OSS output_artifact = couler.create_oss_artifact( path="/home/t1.txt", bucket="test-bucket/", accesskey_id="abcde", accesskey_secret="abc12345", key="osspath/t1", endpoint="xyz.com", ) couler.run_container( image="docker/whalesay:latest", args=["echo -n hello world > %s" % output_artifact.path], command=["bash", "-c"], output=output_artifact, ) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0] self.assertEqual(s.container_spec.image, "docker/whalesay:latest") self.assertTrue(s.outputs[0].artifact.name.startswith("output-oss"))
def test_script_step(self): def echo(): print("echo") couler.run_script( image="docker/whalesay:latest", source=echo, resources={ "cpu": "2", "memory": "1Gi" }, cache=couler.Cache(name="cache-name", key="cache-key", max_age="60s"), ) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] self.assertFalse(s.HasField("resource_spec")) self.assertEqual(s.script, '\nprint("echo")\n') self.assertEqual(s.container_spec.resources["cpu"], "2") self.assertEqual(s.cache.name, "cache-name") self.assertEqual(s.cache.key, "cache-key") self.assertEqual(s.cache.max_age, "60s")
def test_canned_step(self): couler.run_canned_step(name="test", args={"k1": "v1", "k2": "v2"}) proto_wf = get_default_proto_workflow() s = proto_wf.steps[0].steps[0] self.assertEqual(s.canned_step_spec.name, "test") self.assertEqual(s.canned_step_spec.args, {"k1": "v1", "k2": "v2"})