Python lazy_load_module 예제들

예제 #1

def test_lazy_loader_error_message():
    lazy_mod = lazy_loader.lazy_load_module("made.up.module")
    lazy_loader.LazyLoadPlugin("uninstalled_plugin", [], [lazy_mod])
    with pytest.raises(ImportError) as e:
        lazy_mod.some_bad_attr

    assert "uninstalled_plugin" in six.text_type(e.value)
    assert "flytekit[all]" in six.text_type(e.value)

예제 #2

파일: __init__.py 프로젝트: vglocus/flytekit

"""
This file is for old style plugins - for new plugins that work with the Python native-typed Flytekit, please
refer to the plugin specific directory underneath the plugins folder at the top level of this repository.
"""
from flytekit.tools import lazy_loader as _lazy_loader

pyspark = _lazy_loader.lazy_load_module("pyspark")  # type: _lazy_loader._LazyLoadModule

k8s = _lazy_loader.lazy_load_module("k8s")  # type: _lazy_loader._LazyLoadModule
type(k8s).add_sub_module("io.api.core.v1.generated_pb2")
type(k8s).add_sub_module("io.apimachinery.pkg.api.resource.generated_pb2")

flyteidl = _lazy_loader.lazy_load_module("flyteidl")  # type: _lazy_loader._LazyLoadModule
type(flyteidl).add_sub_module("plugins.sidecar_pb2")

numpy = _lazy_loader.lazy_load_module("numpy")  # type: _lazy_loader._LazyLoadModule
pandas = _lazy_loader.lazy_load_module("pandas")  # type: _lazy_loader._LazyLoadModule

hmsclient = _lazy_loader.lazy_load_module("hmsclient")  # type: _lazy_loader._LazyLoadModule
type(hmsclient).add_sub_module("genthrift.hive_metastore.ttypes")

sagemaker_training = _lazy_loader.lazy_load_module("sagemaker_training")  # type: _lazy_loader._LazyLoadModule

papermill = _lazy_loader.lazy_load_module("papermill")  # type: _lazy_loader._LazyLoadModule

_lazy_loader.LazyLoadPlugin("spark", ["pyspark>=2.4.0,<3.0.0"], [pyspark])

_lazy_loader.LazyLoadPlugin("spark3", ["pyspark>=3.0.0"], [pyspark])

_lazy_loader.LazyLoadPlugin("sidecar", ["k8s-proto>=0.0.3,<1.0.0"], [k8s, flyteidl])

예제 #3

from __future__ import absolute_import
from flytekit.tools import lazy_loader as _lazy_loader


pyspark = _lazy_loader.lazy_load_module("pyspark")  # type: types.ModuleType

k8s = _lazy_loader.lazy_load_module("k8s")  # type: types.ModuleType
type(k8s).add_sub_module("io.api.core.v1.generated_pb2")
type(k8s).add_sub_module("io.apimachinery.pkg.api.resource.generated_pb2")

flyteidl = _lazy_loader.lazy_load_module("flyteidl")  # type: types.ModuleType
type(flyteidl).add_sub_module("plugins.sidecar_pb2")

numpy = _lazy_loader.lazy_load_module("numpy")  # type: types.ModuleType
pandas = _lazy_loader.lazy_load_module("pandas")  # type: types.ModuleType

hmsclient = _lazy_loader.lazy_load_module("hmsclient")  # type: types.ModuleType
type(hmsclient).add_sub_module("genthrift.hive_metastore.ttypes")

_lazy_loader.LazyLoadPlugin(
    "spark",
    ["pyspark>=2.4.0,<3.0.0"],
    [pyspark]
)

_lazy_loader.LazyLoadPlugin(
    "sidecar",
    ["k8s-proto>=0.0.2,<1.0.0"],
    [k8s, flyteidl]
)

예제 #4

파일: __init__.py 프로젝트: ybubnov/flytekit

from flytekit.tools import lazy_loader as _lazy_loader

pyspark = _lazy_loader.lazy_load_module(
    "pyspark")  # type: _lazy_loader._LazyLoadModule

k8s = _lazy_loader.lazy_load_module(
    "k8s")  # type: _lazy_loader._LazyLoadModule
type(k8s).add_sub_module("io.api.core.v1.generated_pb2")
type(k8s).add_sub_module("io.apimachinery.pkg.api.resource.generated_pb2")

flyteidl = _lazy_loader.lazy_load_module(
    "flyteidl")  # type: _lazy_loader._LazyLoadModule
type(flyteidl).add_sub_module("plugins.sidecar_pb2")

numpy = _lazy_loader.lazy_load_module(
    "numpy")  # type: _lazy_loader._LazyLoadModule
pandas = _lazy_loader.lazy_load_module(
    "pandas")  # type: _lazy_loader._LazyLoadModule

hmsclient = _lazy_loader.lazy_load_module(
    "hmsclient")  # type: _lazy_loader._LazyLoadModule
type(hmsclient).add_sub_module("genthrift.hive_metastore.ttypes")

torch = _lazy_loader.lazy_load_module(
    "torch")  # type: _lazy_loader._LazyLoadModule

tensorflow = _lazy_loader.lazy_load_module(
    "tensorflow")  # type: _lazy_loader._LazyLoadModule

_lazy_loader.LazyLoadPlugin("spark", ["pyspark>=2.4.0,<3.0.0"], [pyspark])