def test_task_system_failure():
with TemporaryConfiguration(os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../../../common/configs/local.config'),
internal_overrides={
'image':
'myflyteimage:{}'.format(
os.environ.get('IMAGE_VERSION',
'sha')),
'project':
'myflyteproject',
'domain':
'development'
}):
m = MagicMock()
m.execute = _raise_system_exception
with utils.AutoDeletingTempDir("test") as tmp:
engine.FlyteTask(m).execute(None, {'output_prefix': tmp.name})
doc = errors.ErrorDocument.from_flyte_idl(
utils.load_proto_from_file(
errors_pb2.ErrorDocument,
os.path.join(tmp.name, constants.ERROR_FILE_NAME)))
assert doc.error.code == "SYSTEM:Unknown"
assert doc.error.kind == errors.ContainerError.Kind.RECOVERABLE
assert "errorERRORerror" in doc.error.message
def workflows(ctx, version=None, folder=None):
_logging.getLogger().setLevel(_logging.DEBUG)
if folder:
click.echo(f"Writing output to {folder}")
project = ctx.obj[CTX_PROJECT]
domain = ctx.obj[CTX_DOMAIN]
pkgs = ctx.obj[CTX_PACKAGES]
version = (
version
or ctx.obj[CTX_VERSION]
or _internal_configuration.look_up_version_from_image_tag(_internal_configuration.IMAGE.get())
)
internal_settings = {
"project": project,
"domain": domain,
"version": version,
}
# Populate internal settings for project/domain/version from the environment so that the file names are resolved
# with the correct strings. The file itself doesn't need to change though.
with TemporaryConfiguration(_internal_configuration.CONFIGURATION_PATH.get(), internal_settings):
_logging.debug(
"Serializing with settings\n"
"\n Project: {}"
"\n Domain: {}"
"\n Version: {}"
"\n\nover the following packages {}".format(project, domain, version, pkgs)
)
serialize_all(project, domain, pkgs, version, folder)
def test_scope_deprecation():
with TemporaryConfiguration(
os.path.join(os.path.dirname(__file__),
"auth_deprecation.config")):
assert _get_basic_flow_scopes() == ["custom_basic"]
with TemporaryConfiguration(
os.path.join(os.path.dirname(__file__),
"auth_deprecation2.config")):
assert _get_basic_flow_scopes() == [
"custom_basic", "other_scope", "profile"
]
with TemporaryConfiguration(
os.path.join(os.path.dirname(__file__),
"auth_deprecation3.config")):
assert _get_basic_flow_scopes() == ["custom_basic"]
def workflows(ctx):
pkgs = ctx.obj[CTX_PACKAGES]
internal_settings = {
'project': ctx.obj[CTX_PROJECT],
'domain': ctx.obj[CTX_DOMAIN],
'version': ctx.obj[CTX_VERSION]
}
# Populate internal settings for project/domain/version from the environment so that the file names are resolved
# with the correct strings. The file itself doesn't need to change though.
with TemporaryConfiguration(CONFIGURATION_PATH.get(), internal_settings):
serialize_workflows(pkgs)
def temp_config():
with TemporaryConfiguration(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../common/configs/local.config'),
internal_overrides={
'image': 'myflyteimage:{}'.format(
os.environ.get('IMAGE_VERSION', 'sha')
),
'project': 'myflyteproject',
'domain': 'development'
}
):
yield
def test__extract_files(load_mock):
id = _core_identifier.Identifier(_core_identifier.ResourceType.TASK, "myproject", "development", "name", "v")
t = get_sample_task()
with TemporaryConfiguration(
"", internal_overrides={"image": "myflyteimage:v123", "project": "myflyteproject", "domain": "development"},
):
task_spec = t.serialize()
load_mock.side_effect = [id.to_flyte_idl(), task_spec]
new_id, entity = _main._extract_pair("a", "b")
assert new_id == id.to_flyte_idl()
assert task_spec == entity
def temp_config():
with TemporaryConfiguration(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"../../../common/configs/local.config",
),
internal_overrides={
"image": "myflyteimage:{}".format(os.environ.get("IMAGE_VERSION", "sha")),
"project": "myflyteproject",
"domain": "development",
},
):
yield
def test_task_serialization():
t = get_sample_task()
with TemporaryConfiguration(
_os.path.join(_os.path.dirname(_os.path.realpath(__file__)), '../../../common/configs/local.config'),
internal_overrides={
'image': 'myflyteimage:v123',
'project': 'myflyteproject',
'domain': 'development'
}
):
s = t.serialize()
assert isinstance(s, _admin_task_pb2.TaskSpec)
assert s.template.id.name == 'tests.flytekit.unit.common_tests.tasks.test_task.my_task'
assert s.template.container.image == 'myflyteimage:v123'
def test__extract_files(load_mock):
id = _core_identifier.Identifier(_core_identifier.ResourceType.TASK,
'myproject', 'development', 'name', 'v')
t = get_sample_task()
with TemporaryConfiguration("",
internal_overrides={
'image': 'myflyteimage:v123',
'project': 'myflyteproject',
'domain': 'development'
}):
task_spec = t.serialize()
load_mock.side_effect = [id.to_flyte_idl(), task_spec]
new_id, entity = _main._extract_pair('a', 'b')
assert new_id == id.to_flyte_idl()
assert task_spec == entity
def test_task_serialization():
t = get_sample_task()
with TemporaryConfiguration(
_os.path.join(
_os.path.dirname(_os.path.realpath(__file__)),
"../../../common/configs/local.config",
),
internal_overrides={
"image": "myflyteimage:v123",
"project": "myflyteproject",
"domain": "development"
},
):
s = t.serialize()
assert isinstance(s, _admin_task_pb2.TaskSpec)
assert s.template.id.name == "tests.flytekit.unit.common_tests.tasks.test_task.my_task"
assert s.template.container.image == "myflyteimage:v123"
@tfjob_task.tf_job_task(image='ciscoai/mnist:3088D0CF',
num_ps=1,
replicas=2,
command='python /opt/model.py',
args=mnist_trainer_args,
volumeClaimName='mnist-trainer-claim')
def mnist_trainer():
pass
@workflow_class
class DemoWorkflow(object):
mnist_task = mnist_trainer()
if __name__ == '__main__':
_PROJECT = 'demo'
_DOMAIN = 'development'
_USAGE = "Usage:\n\n" \
"\tpython sample.py render_task\n" \
"\tpython sample.py execute <version> <train data path> <validation data path> <hyperparameter json>\n"
with TemporaryConfiguration(
os.path.join(os.path.dirname(__file__), "flyte.config")):
if sys.argv[1] == 'render_task':
DemoWorkflow.validate()
else:
print(_USAGE)
def set_fake_config():
with TemporaryConfiguration(None,
internal_overrides={"image": "fakeimage"}):
yield
def set_fake_config():
with TemporaryConfiguration(None,
internal_overrides={'image': 'fakeimage'}):
yield
def test_spark_yield():
with TemporaryConfiguration(None, internal_overrides={"image": "fakeimage"}):
outputs = spark_yield_task.unit_test(num=1)
dj_spec = outputs[_sdk_constants.FUTURES_FILE_NAME]
print(dj_spec)