def test_module(self, name):
"""pip-installed module can be collected."""
if name == "tests" or name == "conftest" or name.startswith("test_"):
pytest.skip(
"pytest modifies both import mechanisms and module objects,"
" which we can't handle right now")
if six.PY2 and name == "pytest_forked":
pytest.skip(
"pytest_forked insists on having an empty __pycache__,"
" which custom modules ignores, which fails our match check")
if CustomModules.get_module_path(name) in ("built-in", "frozen"):
pytest.skip("built into Python; no module file to collect")
if six.PY2 and (name.startswith("tensorflow_") or name == "torch"):
pytest.skip("takes too long")
custom_modules = _DeployableEntity._custom_modules_as_artifact([name])
self.assert_in_custom_modules(custom_modules, name)
def assert_in_custom_modules(custom_modules, module_name):
module = CustomModules.get_module_path(module_name)
with utils.tempdir() as custom_modules_dir:
with zipfile.ZipFile(custom_modules, "r") as zipf:
zipf.extractall(custom_modules_dir)
# TODO: extract sys.path from _verta_config.py instead of walking
for parent_dir, dirnames, filenames in os.walk(custom_modules_dir):
if os.path.basename(module) in dirnames + filenames:
retrieved_module = os.path.join(
parent_dir,
os.path.basename(module),
)
break
else:
raise ValueError("module not found in custom modules")
if os.path.isfile(module):
assert filecmp.cmp(module, retrieved_module)
else:
utils.assert_dirs_match(module, retrieved_module)
def _custom_modules_as_artifact(paths=None):
if isinstance(paths, six.string_types):
paths = [paths]
# If we include a path that is actually a module, then we _must_ add its parent to the
# adjusted sys.path in the end so that we can re-import with the same name.
forced_local_sys_paths = []
if paths is not None:
new_paths = []
for p in paths:
abspath = os.path.abspath(os.path.expanduser(p))
if CustomModules.is_importable(p):
mod_path = CustomModules.get_module_path(p)
new_paths.append(mod_path)
forced_local_sys_paths.append(os.path.dirname(mod_path))
else:
if os.path.exists(abspath):
new_paths.append(abspath)
else:
raise ValueError(
"custom module {} does not correspond to an existing folder or module"
.format(p))
paths = new_paths
forced_local_sys_paths = sorted(list(set(forced_local_sys_paths)))
# collect local sys paths
local_sys_paths = copy.copy(sys.path)
## replace empty first element with cwd
## https://docs.python.org/3/library/sys.html#sys.path
if local_sys_paths[0] == "":
local_sys_paths[0] = os.getcwd()
## convert to absolute paths
local_sys_paths = list(map(os.path.abspath, local_sys_paths))
## remove paths that don't exist
local_sys_paths = list(filter(os.path.exists, local_sys_paths))
## remove .ipython
local_sys_paths = list(
filter(lambda path: not path.endswith(".ipython"),
local_sys_paths))
## remove virtual (and real) environments
local_sys_paths = list(
filter(lambda path: not _utils.is_in_venv(path), local_sys_paths))
# get paths to files within
if paths is None:
# Python files within filtered sys.path dirs
paths = local_sys_paths
extensions = ['py', 'pyc', 'pyo']
else:
# all user-specified files
paths = paths
extensions = None
local_filepaths = _utils.find_filepaths(
paths,
extensions=extensions,
include_hidden=True,
include_venv=False, # ignore virtual environments nested within
)
## remove .git
local_filepaths = set(
filter(
lambda path: not path.endswith(".git") and ".git/" not in path,
local_filepaths))
# obtain deepest common directory
# This directory on the local system will be mirrored in `_CUSTOM_MODULES_DIR` in
# deployment.
curr_dir = os.path.join(os.getcwd(), "")
paths_plus = list(local_filepaths) + [curr_dir]
common_prefix = os.path.commonprefix(paths_plus)
common_dir = os.path.dirname(common_prefix)
# replace `common_dir` with `_CUSTOM_MODULES_DIR` for deployment sys.path
depl_sys_paths = list(
map(lambda path: os.path.relpath(path, common_dir),
local_sys_paths + forced_local_sys_paths))
depl_sys_paths = list(
map(lambda path: os.path.join(_CUSTOM_MODULES_DIR, path),
depl_sys_paths))
bytestream = six.BytesIO()
with zipfile.ZipFile(bytestream, 'w') as zipf:
for filepath in local_filepaths:
arcname = os.path.relpath(
filepath, common_dir) # filepath relative to archive root
try:
zipf.write(filepath, arcname)
except:
# maybe file has corrupt metadata; try reading then writing contents
with open(filepath, 'rb') as f:
zipf.writestr(
_artifact_utils.global_read_zipinfo(arcname),
f.read(),
)
# add verta config file for sys.path and chdir
working_dir = os.path.join(_CUSTOM_MODULES_DIR,
os.path.relpath(curr_dir, common_dir))
zipf.writestr(
_artifact_utils.global_read_zipinfo("_verta_config.py"),
six.ensure_binary('\n'.join([
"import os, sys",
"",
"",
"sys.path = sys.path[:1] + {} + sys.path[1:]".format(
depl_sys_paths),
"",
"try:",
" os.makedirs(\"{}\")".format(working_dir),
"except OSError: # already exists",
" pass",
"os.chdir(\"{}\")".format(working_dir),
])))
# add __init__.py
init_filename = "__init__.py"
if init_filename not in zipf.namelist():
zipf.writestr(
_artifact_utils.global_read_zipinfo(init_filename),
b"",
)
bytestream.seek(0)
return bytestream