def test_workflow_name_salt_applies(self):
couler.config_defaults(name_salter=lambda name: "%s-salted" % name)
couler.config_workflow(name="test_name_salter_applies")
ret = couler.workflow_yaml()
self.assertEqual(
ret["metadata"]["name"], "test_name_salter_applies-salted"
)
def test_workflow_config(self):
flip_coin()
tails()
couler.config_workflow(
name="test-workflow", user_id="88888888", time_to_clean=100
)
wf = couler.workflow_yaml()
expected_meta = {
"name": "test-workflow",
# "labels": {"couler_job_user": "******"},
}
self.assertEqual(wf["metadata"], expected_meta)
def test_cluster_config(self):
couler.config_workflow(cluster_config_file=os.path.join(
os.path.dirname(__file__), "test_data/dummy_cluster_config.py"))
couler.run_container(
image="docker/whalesay:latest",
args=["echo -n hello world"],
command=["bash", "-c"],
step_name="A",
)
wf = couler.workflow_yaml()
self.assertTrue(wf["spec"]["hostNetwork"])
self.assertEqual(wf["spec"]["templates"][1]["tolerations"], [])
couler._cleanup()
def test_workflow_service_account(self):
self.assertIsNone(couler.workflow.service_account)
flip_coin()
self.assertNotIn("serviceAccountName", couler.workflow_yaml()["spec"])
couler.config_workflow(service_account="test-serviceaccount")
self.assertEqual(couler.workflow.service_account,
"test-serviceaccount")
self.assertEqual(
couler.workflow_yaml()["spec"]["serviceAccountName"],
"test-serviceaccount",
)
couler._cleanup()
self.assertIsNone(couler.workflow.service_account)
def test_cron_workflow(self):
def tails():
return couler.run_container(
image="python:3.6",
command=["bash", "-c", 'echo "run schedule job"'],
)
tails()
# schedule to run at one minute past midnight (00:01) every day
cron_config = {"schedule": "1 0 * * *", "suspend": "false"}
couler.config_workflow(name="pytest", cron_config=cron_config)
self.check_argo_yaml("cron_workflow_golden.yaml")
def flip_coin():
return couler.run_script(image="python:alpine3.6", source=random_code)
def heads():
return couler.run_container(image="alpine:3.6",
command=["sh", "-c", 'echo "it was heads"'])
def tails():
return couler.run_container(image="alpine:3.6",
command=["sh", "-c", 'echo "it was tails"'])
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting flip coin example workflow via %s implementation" %
impl_type)
couler.config_workflow(
name="flip-coin-%s" % impl_type.lower(),
timeout=3600,
time_to_clean=3600 * 1.5,
)
result = flip_coin()
couler.when(couler.equal(result, "heads"), lambda: heads())
couler.when(couler.equal(result, "tails"), lambda: tails())
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from collections import OrderedDict
import couler.argo as couler
from couler.tests.argo_yaml_test import ArgoYamlTest
_test_data_dir = "test_data"
couler.config_workflow(name="pytest")
def random_code():
import random
result = "heads" if random.randint(0, 1) == 0 else "tails"
print(result)
def flip_coin():
return couler.run_script(image="python:3.6", source=random_code)
def heads():
return couler.run_container(
def flip_coin():
return couler.run_script(image="python:alpine3.6", source=random_code)
def heads():
return couler.run_container(
image="alpine:3.6", command=["sh", "-c", 'echo "it was heads"']
)
def tails():
return couler.run_container(
image="alpine:3.6", command=["sh", "-c", 'echo "it was tails"']
)
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting workflow via %s implementation" % impl_type)
couler.config_workflow(timeout=3600, time_to_clean=3600 * 1.5)
result = flip_coin()
couler.when(couler.equal(result, "heads"), lambda: heads())
couler.when(couler.equal(result, "tails"), lambda: tails())
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)
print("Workflow submitted for flip coin example")
# limitations under the License.
import os
import couler.argo as couler
from couler.argo_submitter import (
_SUBMITTER_IMPL_ENV_VAR_KEY,
ArgoSubmitter,
_SubmitterImplTypes,
)
from couler.steps import mpi
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting DAG example workflow via %s implementation" %
impl_type)
couler.config_workflow(
"mpijob-%s" % impl_type.lower(),
timeout=3600,
time_to_clean=3600 * 1.5,
)
mpi.train(
image="alpine:3.6",
command=["sh", "-c", 'echo "running"; exit 0'],
num_workers=1,
)
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)
import os
import couler.argo as couler
from couler.argo_submitter import (
_SUBMITTER_IMPL_ENV_VAR_KEY,
ArgoSubmitter,
_SubmitterImplTypes,
)
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting memoization example workflow via %s implementation" %
impl_type)
couler.config_workflow(
name="memoization-%s" % impl_type.lower(),
timeout=3600,
time_to_clean=3600 * 1.5,
)
couler.run_container(
image="alpine:3.6",
command=["sh", "-c", 'echo "Hello world"'],
cache=couler.Cache(name="cache-name",
key="cache-key",
max_age="60s"),
)
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)
