def test_task_system_failure(): with TemporaryConfiguration(os.path.join( os.path.dirname(os.path.realpath(__file__)), '../../../common/configs/local.config'), internal_overrides={ 'image': 'myflyteimage:{}'.format( os.environ.get('IMAGE_VERSION', 'sha')), 'project': 'myflyteproject', 'domain': 'development' }): m = MagicMock() m.execute = _raise_system_exception with utils.AutoDeletingTempDir("test") as tmp: engine.FlyteTask(m).execute(None, {'output_prefix': tmp.name}) doc = errors.ErrorDocument.from_flyte_idl( utils.load_proto_from_file( errors_pb2.ErrorDocument, os.path.join(tmp.name, constants.ERROR_FILE_NAME))) assert doc.error.code == "SYSTEM:Unknown" assert doc.error.kind == errors.ContainerError.Kind.RECOVERABLE assert "errorERRORerror" in doc.error.message
def workflows(ctx, version=None, folder=None): _logging.getLogger().setLevel(_logging.DEBUG) if folder: click.echo(f"Writing output to {folder}") project = ctx.obj[CTX_PROJECT] domain = ctx.obj[CTX_DOMAIN] pkgs = ctx.obj[CTX_PACKAGES] version = ( version or ctx.obj[CTX_VERSION] or _internal_configuration.look_up_version_from_image_tag(_internal_configuration.IMAGE.get()) ) internal_settings = { "project": project, "domain": domain, "version": version, } # Populate internal settings for project/domain/version from the environment so that the file names are resolved # with the correct strings. The file itself doesn't need to change though. with TemporaryConfiguration(_internal_configuration.CONFIGURATION_PATH.get(), internal_settings): _logging.debug( "Serializing with settings\n" "\n Project: {}" "\n Domain: {}" "\n Version: {}" "\n\nover the following packages {}".format(project, domain, version, pkgs) ) serialize_all(project, domain, pkgs, version, folder)
def test_scope_deprecation(): with TemporaryConfiguration( os.path.join(os.path.dirname(__file__), "auth_deprecation.config")): assert _get_basic_flow_scopes() == ["custom_basic"] with TemporaryConfiguration( os.path.join(os.path.dirname(__file__), "auth_deprecation2.config")): assert _get_basic_flow_scopes() == [ "custom_basic", "other_scope", "profile" ] with TemporaryConfiguration( os.path.join(os.path.dirname(__file__), "auth_deprecation3.config")): assert _get_basic_flow_scopes() == ["custom_basic"]
def workflows(ctx): pkgs = ctx.obj[CTX_PACKAGES] internal_settings = { 'project': ctx.obj[CTX_PROJECT], 'domain': ctx.obj[CTX_DOMAIN], 'version': ctx.obj[CTX_VERSION] } # Populate internal settings for project/domain/version from the environment so that the file names are resolved # with the correct strings. The file itself doesn't need to change though. with TemporaryConfiguration(CONFIGURATION_PATH.get(), internal_settings): serialize_workflows(pkgs)
def temp_config(): with TemporaryConfiguration( os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../common/configs/local.config'), internal_overrides={ 'image': 'myflyteimage:{}'.format( os.environ.get('IMAGE_VERSION', 'sha') ), 'project': 'myflyteproject', 'domain': 'development' } ): yield
def test__extract_files(load_mock): id = _core_identifier.Identifier(_core_identifier.ResourceType.TASK, "myproject", "development", "name", "v") t = get_sample_task() with TemporaryConfiguration( "", internal_overrides={"image": "myflyteimage:v123", "project": "myflyteproject", "domain": "development"}, ): task_spec = t.serialize() load_mock.side_effect = [id.to_flyte_idl(), task_spec] new_id, entity = _main._extract_pair("a", "b") assert new_id == id.to_flyte_idl() assert task_spec == entity
def temp_config(): with TemporaryConfiguration( os.path.join( os.path.dirname(os.path.realpath(__file__)), "../../../common/configs/local.config", ), internal_overrides={ "image": "myflyteimage:{}".format(os.environ.get("IMAGE_VERSION", "sha")), "project": "myflyteproject", "domain": "development", }, ): yield
def test_task_serialization(): t = get_sample_task() with TemporaryConfiguration( _os.path.join(_os.path.dirname(_os.path.realpath(__file__)), '../../../common/configs/local.config'), internal_overrides={ 'image': 'myflyteimage:v123', 'project': 'myflyteproject', 'domain': 'development' } ): s = t.serialize() assert isinstance(s, _admin_task_pb2.TaskSpec) assert s.template.id.name == 'tests.flytekit.unit.common_tests.tasks.test_task.my_task' assert s.template.container.image == 'myflyteimage:v123'
def test__extract_files(load_mock): id = _core_identifier.Identifier(_core_identifier.ResourceType.TASK, 'myproject', 'development', 'name', 'v') t = get_sample_task() with TemporaryConfiguration("", internal_overrides={ 'image': 'myflyteimage:v123', 'project': 'myflyteproject', 'domain': 'development' }): task_spec = t.serialize() load_mock.side_effect = [id.to_flyte_idl(), task_spec] new_id, entity = _main._extract_pair('a', 'b') assert new_id == id.to_flyte_idl() assert task_spec == entity
def test_task_serialization(): t = get_sample_task() with TemporaryConfiguration( _os.path.join( _os.path.dirname(_os.path.realpath(__file__)), "../../../common/configs/local.config", ), internal_overrides={ "image": "myflyteimage:v123", "project": "myflyteproject", "domain": "development" }, ): s = t.serialize() assert isinstance(s, _admin_task_pb2.TaskSpec) assert s.template.id.name == "tests.flytekit.unit.common_tests.tasks.test_task.my_task" assert s.template.container.image == "myflyteimage:v123"
@tfjob_task.tf_job_task(image='ciscoai/mnist:3088D0CF', num_ps=1, replicas=2, command='python /opt/model.py', args=mnist_trainer_args, volumeClaimName='mnist-trainer-claim') def mnist_trainer(): pass @workflow_class class DemoWorkflow(object): mnist_task = mnist_trainer() if __name__ == '__main__': _PROJECT = 'demo' _DOMAIN = 'development' _USAGE = "Usage:\n\n" \ "\tpython sample.py render_task\n" \ "\tpython sample.py execute <version> <train data path> <validation data path> <hyperparameter json>\n" with TemporaryConfiguration( os.path.join(os.path.dirname(__file__), "flyte.config")): if sys.argv[1] == 'render_task': DemoWorkflow.validate() else: print(_USAGE)
def set_fake_config(): with TemporaryConfiguration(None, internal_overrides={"image": "fakeimage"}): yield
def set_fake_config(): with TemporaryConfiguration(None, internal_overrides={'image': 'fakeimage'}): yield
def test_spark_yield(): with TemporaryConfiguration(None, internal_overrides={"image": "fakeimage"}): outputs = spark_yield_task.unit_test(num=1) dj_spec = outputs[_sdk_constants.FUTURES_FILE_NAME] print(dj_spec)