def _update_namespaces(self) -> None:
"""Get the available namespaces.
"""
# user defined search paths
dir_list = set()
self.namespaces = dict()
for k, v in settings['SEARCH_PATHS'].items():
if os.path.isdir(v):
meta_path = os.path.join(v,
settings['NAMESPACE_CONFIG_FILENAME'])
meta = U.load_yaml(meta_path)[0] if os.path.exists(
meta_path) else dict()
meta['module_path'] = os.path.abspath(
os.path.join(v, meta.get('module_path', './')))
if os.path.isdir(meta['module_path']):
self.namespaces[k] = meta
dir_list.add(meta['module_path'])
else:
U.alert_msg(
'Namespace "%s" has an invalid module path "%s".' %
(k, meta['module_path']))
# current path
current_path = os.path.abspath(os.curdir)
if not current_path in dir_list:
self.namespaces['main'] = dict(module_path=current_path)
def _install_namespaces_from_path(path: str,
namespace: Optional[str] = None) -> None:
"""Install namespaces from path.
Parameters:
path:
Path to the directory
namespace:
Specified namespace
"""
path = os.path.abspath(path)
namespace = namespace or ModuleManager._format_namespace(
os.path.basename(path))
search_paths = settings['SEARCH_PATHS']
for k, v in search_paths.items():
if namespace == k:
U.alert_msg(
'Namespace "%s" is already bound to the path "%s".' %
(k, v))
return
if path == v:
U.alert_msg(
'"%s" is already installed under the namespace "%s".' %
(v, k))
return
search_paths[namespace] = path
settings['SEARCH_PATHS'] = search_paths
settings.save()
def _remove_namespaces_from_path(src: str) -> Optional[str]:
"""Remove namespaces from path.
Parameters:
src:
Namespace or path
"""
if os.path.isdir(src):
path, namespace = os.path.abspath(src), None
else:
path, namespace = None, src
delete_key = None
search_paths = settings['SEARCH_PATHS']
for k, v in search_paths.items():
if namespace == k:
delete_key = k
break
if path == v:
delete_key = k
break
if delete_key is None:
if namespace:
U.alert_msg('The namespace "%s" is not installed.' % namespace)
if path:
U.alert_msg('The path "%s" is not installed.' % path)
else:
path = search_paths.pop(delete_key)
settings['SEARCH_PATHS'] = search_paths
settings.save()
return path
def _import_nest_modules_from_py_module(
namespace: str, py_module: object,
nest_modules: Dict[str, object]) -> bool:
"""Import registered Nest modules from a given python module.
Parameters:
namespace:
A namespace that is used to avoid name conflicts
py_module:
The python module
nest_modules:
The dict for storing Nest modules
Returns:
The id of imported Nest modules
"""
imported_ids = []
# search for Nest modules
for key, val in py_module.__dict__.items():
module_id = U.encode_id(namespace, key)
if not key.startswith('_') and type(val).__name__ == 'NestModule':
if module_id in nest_modules.keys():
U.alert_msg('There are duplicate "%s" modules under namespace "%s".' % \
(key, namespace))
else:
nest_modules[module_id] = val
imported_ids.append(module_id)
return imported_ids
def install_dep(url, tool):
# filter deps
if re.match(r'^[a-zA-Z0-9<=>.-]+$', dep):
try:
subprocess.check_call([
sys.executable, '-m', tool, 'install', dep
])
except subprocess.CalledProcessError:
U.alert_msg(
'Failed to install "%s" for "%s". Please manually install it.'
% (dep, dirname))
def _install_namespaces_from_url(url: str,
namespace: Optional[str] = None) -> None:
"""Install namespaces from url.
Parameters:
url:
URL of the zip file or git repo
namespace:
Specified namespace
"""
# pre-process short URL
if url.startswith('github@'):
m = re.match(r'^github@([\w\-\_]+)/([\w\-\_]+)(:[\w\-\_]+)*$', url)
repo = m.group(1) + '/' + m.group(2)
branch = m.group(3) or ':master'
url = '-b %s https://github.com/%s.git' % (branch[1:], repo)
elif url.startswith('gitlab@'):
m = re.match(r'^gitlab@([\w\-\_]+)/([\w\-\_]+)(:[\w\-\_]+)*$', url)
repo = m.group(1) + '/' + m.group(2)
branch = m.group(3) or ':master'
url = '-b %s https://gitlab.com/%s.git' % (branch[1:], repo)
elif url.startswith('bitbucket@'):
m = re.match(r'^bitbucket@([\w\-\_]+)/([\w\-\_]+)(:[\w\-\_]+)*$',
url)
repo = m.group(1) + '/' + m.group(2)
branch = m.group(3) or ':master'
url = '-b %s https://bitbucket.org/%s.git' % (branch[1:], repo)
elif url.startswith('file@'):
path = url[5:]
url = 'file:///' + os.path.abspath(path)
for dirname in ModuleManager._fetch_nest_modules_from_url(url, './'):
module_path = os.path.join('./', dirname)
ModuleManager._install_namespaces_from_path(module_path, namespace)
# parse config
meta_path = os.path.join(module_path,
settings['NAMESPACE_CONFIG_FILENAME'])
meta = U.load_yaml(meta_path)[0] if os.path.exists(
meta_path) else dict()
if settings['AUTO_INSTALL_REQUIREMENTS']:
# auto install deps
for dep in meta.get('requirements', []):
# helper function
def install_dep(url, tool):
# filter deps
if re.match(r'^[a-zA-Z0-9<=>.-]+$', dep):
try:
subprocess.check_call([
sys.executable, '-m', tool, 'install', dep
])
except subprocess.CalledProcessError:
U.alert_msg(
'Failed to install "%s" for "%s". Please manually install it.'
% (dep, dirname))
if isinstance(dep, str):
# use pip by default
install_dep(dep, 'pip')
elif isinstance(dep,
dict) and 'url' in dep and 'tool' in dep:
install_dep(dep['url'], dep['tool'])
else:
U.alert_msg('Invalid install requirement "%s".' % dep)
def _fetch_nest_modules_from_url(url: str, dst: str) -> None:
"""Fetch and unzip Nest modules from url.
Parameters:
url:
URL of the zip file or git repo
dst:
Save dir path
"""
def _hook(count, block_size, total_size):
size = float(count * block_size) / (1024.0 * 1024.0)
total_size = float(total_size / (1024.0 * 1024.0))
if total_size > 0:
size = min(size, total_size)
percent = 100.0 * size / total_size
sys.stdout.write("\rFetching...%d%%, %.2f MB / %.2f MB" %
(percent, size, total_size))
else:
sys.stdout.write("\rFetching...%.2f MB" % size)
sys.stdout.flush()
# extract
if url.endswith('zip'):
import random
import string
import zipfile
from urllib import request, error
cache_name = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(6)) + '.cache'
cache_path = os.path.join(dst, cache_name)
try:
# download
request.urlretrieve(url, cache_path, _hook)
sys.stdout.write('\n')
# unzip
with zipfile.ZipFile(cache_path, 'r') as f:
file_list = f.namelist()
namespaces = set([v.split('/')[0] for v in file_list])
members = [v for v in file_list if '/' in v]
f.extractall(dst, members)
return namespaces
except error.URLError as exc_info:
U.alert_msg('Could not fetch "%s". %s' % (url, exc_info))
return []
except Exception as exc_info:
U.alert_msg('Error occurs during extraction. %s' % exc_info)
return []
finally:
# remove cache
if os.path.exists(cache_path):
os.remove(cache_path)
elif url.endswith('.git'):
try:
repo_name = url[url.rfind('/') + 1:-4]
match = re.search(r'(?:\s|^)(?:-b|--branch) (\w+)', url)
if match:
repo_name += '-' + match.group(1)
subprocess.check_call(['git', 'clone'] + url.split() +
[repo_name])
return [repo_name]
except subprocess.CalledProcessError as exc_info:
U.alert_msg('Failed to clone "%s".' % url)
return []
else:
raise NotImplementedError(
'Only supports zip file and git repo for now. Got "%s".' % url)
def _import_nest_modules_from_file(
path: str,
namespace: str,
py_modules: Dict[str, float],
nest_modules: Dict[str, object],
meta: Dict[str, object] = dict()) -> None:
"""Import registered Nest modules form a given file.
Parameters:
path:
The path to the file
namespace:
A namespace that is used to avoid name conflicts
py_modules:
The dict for storing python modules information
nest_modules:
The dict for storing Nest modules
meta:
Global meta information
"""
py_module_name = os.path.basename(path).split('.')[0]
py_module_id = U.encode_id(namespace, py_module_name)
timestamp = os.path.getmtime(path)
# check whether the python module have already been imported
is_reload = False
if py_module_id in py_modules.keys():
if timestamp <= py_modules[py_module_id][0]:
# skip
return
else:
is_reload = True
# import the python module
# note that a python module could contain multiple Nest modules.
ref_id = 'nest.' + namespace + '.' + py_module_name
spec = importlib.util.spec_from_file_location(ref_id, path)
if spec is not None:
py_module = importlib.util.module_from_spec(spec)
py_module.__nest_meta__ = U.merge_dict(dict(), meta, union=True)
# no need to bind global requirements to individual Nest modules.
requirements = py_module.__nest_meta__.pop('requirements', None)
if requirements is not None:
requirements = [
dict(url=v, tool='pip') if isinstance(v, str) else v
for v in requirements
]
sys.modules[ref_id] = py_module
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
spec.loader.exec_module(py_module)
except Exception as exc_info:
# helper function
def find_requirement(name):
if isinstance(requirements,
list) and len(requirements) > 0:
scores = [(SequenceMatcher(None, name,
v['url']).ratio(), v)
for v in requirements]
return max(scores, key=lambda x: x[0])
# install tip
tip = ''
if (type(exc_info) is ImportError or type(exc_info) is
ModuleNotFoundError) and exc_info.name is not None:
match = find_requirement(exc_info.name)
if match and match[0] > settings['INSTALL_TIP_THRESHOLD']:
tip = 'Try to execute "%s install %s" to install the missing dependency.' % \
(match[1]['tool'], match[1]['url'])
exc_info = str(exc_info)
exc_info = exc_info if exc_info.endswith(
'.') else exc_info + '.'
U.alert_msg(
'%s The package "%s" under namespace "%s" could not be imported. %s'
% (exc_info, py_module_name, namespace, tip))
else:
# remove old Nest modules
if is_reload:
for key in py_modules[py_module_id][1]:
if key in nest_modules.keys():
del nest_modules[key]
# import all Nest modules within the python module
imported_ids = ModuleManager._import_nest_modules_from_py_module(
namespace, py_module, nest_modules)
if len(imported_ids) > 0:
# record modified time, id, and spec of imported Nest modules
py_modules[py_module_id] = (timestamp, imported_ids,
py_module.__spec__)