def test_module_and_local_dir_have_same_name(self, worker_id):
"""If a pip-installed module and a local directory share a name, the module is collected.
If a user can import a package "foo" in their environment, and uses
custom modules to find "foo", we will prefer that package over a
directory/file "foo" in the cwd. Otherwise, it is very difficult or
impossible to force the installed package.
"""
name = worker_id
# avoid using an existing package name
hypothesis.assume(not CustomModules.is_importable(name))
with utils.chtempdir():
# create local directory with same name as package
local_dir = os.path.abspath(name)
os.mkdir(local_dir)
with open(os.path.join(local_dir, "empty.json"), "w") as f:
json.dump({}, f)
# create package in another directory and install
with utils.tempdir() as tempd:
with contexts.installable_package(name, dir=tempd) as pkg_dir:
with contexts.installed_local_package(pkg_dir, name):
# collect and validate custom modules
custom_modules = _DeployableEntity._custom_modules_as_artifact(
[name], )
self.assert_in_custom_modules(custom_modules, name)
def test_deploy_module_and_local_pkg_have_same_name(
self,
deployable_entity,
endpoint,
worker_id,
):
"""Deployment analogue to ``custom_modules/test_custom_modules.py::TestPipInstalledModule::test_module_and_local_pkg_have_same_name``.
A success means that custom modules successfully collected the
pip-installed module, and the deployed model could import it.
"""
name = worker_id
# avoid using an existing package name
hypothesis.assume(not CustomModules.is_importable(name))
with utils.chtempdir():
# create package in *current* directory and install
with contexts.installable_package(name, dir=".") as pkg_dir:
with contexts.installed_local_package(pkg_dir, name):
Model = models.create_custom_module_model(name)
assert Model().predict("") == name # sanity check
deployable_entity.log_model(Model, custom_modules=[name])
deployable_entity.log_environment(Python([]))
endpoint.update(deployable_entity, wait=True)
assert endpoint.get_deployed_model().predict("") == name
def test_deploy_pip_installed_custom_modules(
self,
deployable_entity,
endpoint,
worker_id,
):
"""A pip-installed custom module can be specified by name (instead of path).
A success means that custom modules successfully collected the
pip-installed module, and the deployed model could import it.
"""
name = worker_id
# avoid using an existing package nam
hypothesis.assume(not CustomModules.is_importable(name))
with utils.tempdir() as tempd:
# create package in another directory and install
with contexts.installable_package(name, dir=tempd) as pkg_dir:
with contexts.installed_local_package(pkg_dir, name):
Model = models.create_custom_module_model(name)
assert Model().predict("") == name # sanity check
deployable_entity.log_model(Model, custom_modules=[name])
deployable_entity.log_environment(Python([]))
endpoint.update(deployable_entity, wait=True)
assert endpoint.get_deployed_model().predict("") == name
def installed_local_package(pkg_dir, name):
"""pip install a locally-defined package.
Parameters
----------
pkg_dir : str
Absolute path to the package.
name : str
Name of the package. This is used to uninstall the package on
cleanup.
Warnings
--------
While packages are uninstalled on cleanup, their dependencies might not be.
"""
subprocess.check_call(
[
sys.executable,
"-m",
"pip",
"--no-python-version-warning",
"install",
"-qq",
pkg_dir,
],
)
assert CustomModules.is_importable(name) # verify installation
try:
yield
finally:
subprocess.check_call(
[
sys.executable,
"-m",
"pip",
"--no-python-version-warning",
"uninstall",
"-y",
name,
],
)
# delete cached module to disallow subsequent imports
if name in sys.modules:
del sys.modules[name]
def test_module_and_local_pkg_have_same_name(self, worker_id):
"""A specific case of :meth:`test_module_and_local_dir_have_same_name`.
The local directory *is* a Python package repository (but not directly
importable without ``cd``ing one level into it).
A user may have a monolithic project with model management scripts
alongside Python package directories (that may *also* be installed
into the environment).
"""
name = worker_id
# avoid using an existing package name
hypothesis.assume(not CustomModules.is_importable(name))
with utils.chtempdir():
# create package in *current* directory and install
with contexts.installable_package(name, dir=".") as pkg_dir:
with contexts.installed_local_package(pkg_dir, name):
# collect and validate custom modules
custom_modules = _DeployableEntity._custom_modules_as_artifact(
[name], )
self.assert_in_custom_modules(custom_modules, name)
def _custom_modules_as_artifact(paths=None):
if isinstance(paths, six.string_types):
paths = [paths]
# If we include a path that is actually a module, then we _must_ add its parent to the
# adjusted sys.path in the end so that we can re-import with the same name.
forced_local_sys_paths = []
if paths is not None:
new_paths = []
for p in paths:
abspath = os.path.abspath(os.path.expanduser(p))
if CustomModules.is_importable(p):
mod_path = CustomModules.get_module_path(p)
new_paths.append(mod_path)
forced_local_sys_paths.append(os.path.dirname(mod_path))
else:
if os.path.exists(abspath):
new_paths.append(abspath)
else:
raise ValueError(
"custom module {} does not correspond to an existing folder or module"
.format(p))
paths = new_paths
forced_local_sys_paths = sorted(list(set(forced_local_sys_paths)))
# collect local sys paths
local_sys_paths = copy.copy(sys.path)
## replace empty first element with cwd
## https://docs.python.org/3/library/sys.html#sys.path
if local_sys_paths[0] == "":
local_sys_paths[0] = os.getcwd()
## convert to absolute paths
local_sys_paths = list(map(os.path.abspath, local_sys_paths))
## remove paths that don't exist
local_sys_paths = list(filter(os.path.exists, local_sys_paths))
## remove .ipython
local_sys_paths = list(
filter(lambda path: not path.endswith(".ipython"),
local_sys_paths))
## remove virtual (and real) environments
local_sys_paths = list(
filter(lambda path: not _utils.is_in_venv(path), local_sys_paths))
# get paths to files within
if paths is None:
# Python files within filtered sys.path dirs
paths = local_sys_paths
extensions = ['py', 'pyc', 'pyo']
else:
# all user-specified files
paths = paths
extensions = None
local_filepaths = _utils.find_filepaths(
paths,
extensions=extensions,
include_hidden=True,
include_venv=False, # ignore virtual environments nested within
)
## remove .git
local_filepaths = set(
filter(
lambda path: not path.endswith(".git") and ".git/" not in path,
local_filepaths))
# obtain deepest common directory
# This directory on the local system will be mirrored in `_CUSTOM_MODULES_DIR` in
# deployment.
curr_dir = os.path.join(os.getcwd(), "")
paths_plus = list(local_filepaths) + [curr_dir]
common_prefix = os.path.commonprefix(paths_plus)
common_dir = os.path.dirname(common_prefix)
# replace `common_dir` with `_CUSTOM_MODULES_DIR` for deployment sys.path
depl_sys_paths = list(
map(lambda path: os.path.relpath(path, common_dir),
local_sys_paths + forced_local_sys_paths))
depl_sys_paths = list(
map(lambda path: os.path.join(_CUSTOM_MODULES_DIR, path),
depl_sys_paths))
bytestream = six.BytesIO()
with zipfile.ZipFile(bytestream, 'w') as zipf:
for filepath in local_filepaths:
arcname = os.path.relpath(
filepath, common_dir) # filepath relative to archive root
try:
zipf.write(filepath, arcname)
except:
# maybe file has corrupt metadata; try reading then writing contents
with open(filepath, 'rb') as f:
zipf.writestr(
_artifact_utils.global_read_zipinfo(arcname),
f.read(),
)
# add verta config file for sys.path and chdir
working_dir = os.path.join(_CUSTOM_MODULES_DIR,
os.path.relpath(curr_dir, common_dir))
zipf.writestr(
_artifact_utils.global_read_zipinfo("_verta_config.py"),
six.ensure_binary('\n'.join([
"import os, sys",
"",
"",
"sys.path = sys.path[:1] + {} + sys.path[1:]".format(
depl_sys_paths),
"",
"try:",
" os.makedirs(\"{}\")".format(working_dir),
"except OSError: # already exists",
" pass",
"os.chdir(\"{}\")".format(working_dir),
])))
# add __init__.py
init_filename = "__init__.py"
if init_filename not in zipf.namelist():
zipf.writestr(
_artifact_utils.global_read_zipinfo(init_filename),
b"",
)
bytestream.seek(0)
return bytestream