def test_migrate_config(td):
profile = pjoin(td, 'profile')
jpy = pjoin(td, 'jupyter_config')
ensure_dir_exists(profile)
env = {
'profile': profile,
'jupyter_config': jpy,
}
cfg_py = pjoin(profile, 'ipython_test_config.py')
touch(cfg_py, 'c.Klass.trait = 5\n')
empty_cfg_py = pjoin(profile, 'ipython_empty_config.py')
touch(empty_cfg_py, '# c.Klass.trait = 5\n')
assert not migrate_config('empty', env)
assert not os.path.exists(jpy)
with patch.dict(migrate_mod.config_substitutions, {
re.compile(r'\bKlass\b'): 'Replaced',
}):
assert migrate_config('test', env)
assert os.path.isdir(jpy)
assert sorted(os.listdir(jpy)) == [
'jupyter_test_config.py',
]
with open(pjoin(jpy, 'jupyter_test_config.py')) as f:
text = f.read()
assert text == 'c.Replaced.trait = 5\n'
def test_migrate_config(td):
profile = pjoin(td, 'profile')
jpy = pjoin(td, 'jupyter_config')
ensure_dir_exists(profile)
env = {
'profile': profile,
'jupyter_config': jpy,
}
cfg_py = pjoin(profile, 'ipython_test_config.py')
touch(cfg_py, 'c.Klass.trait = 5\n')
empty_cfg_py = pjoin(profile, 'ipython_empty_config.py')
touch(empty_cfg_py, '# c.Klass.trait = 5\n')
assert not migrate_config('empty', env)
assert not os.path.exists(jpy)
with patch.dict(migrate_mod.config_substitutions, {
re.compile(r'\bKlass\b'): 'Replaced',
}):
assert migrate_config('test', env)
assert os.path.isdir(jpy)
assert sorted(os.listdir(jpy)) == [
'jupyter_test_config.py',
]
with open(pjoin(jpy, 'jupyter_test_config.py')) as f:
text = f.read()
assert text == 'c.Replaced.trait = 5\n'
def launch(image, cwd):
d = os.path.join(jupyter_runtime_dir(), 'docker_kernels')
ensure_dir_exists(d)
set_sticky_bit(d)
conn_file_tmpdir = TemporaryDirectory(dir=d)
conn_info = make_connection_file(conn_file_tmpdir.name)
container = docker.from_env().containers.run(image,
detach=True,
volumes={
conn_file_tmpdir.name: {
'bind': '/connect',
'mode': 'rw'
},
cwd: {
'bind': '/working',
'mode': 'rw'
},
})
container.reload() # Need this to get the IP address
ip = container.attrs['NetworkSettings']['Networks']['bridge']['IPAddress']
if not ip:
raise RuntimeError("No IP address for docker container")
print(container.attrs['NetworkSettings']['Networks'])
conn_info['ip'] = ip
return conn_info, DockerKernelManager(container, conn_file_tmpdir)
def checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
checkpoint_dir = self.get_checkpoints_path_for_file(path)
with self.perm_to_403():
ensure_dir_exists(checkpoint_dir)
cp_path = os.path.join(checkpoint_dir, checkpoint_id)
return cp_path
def _template_paths(self, prune=True, root_dirs=None):
paths = []
root_dirs = self.get_prefix_root_dirs()
template_names = self.get_template_names()
for template_name in template_names:
for base_dir in self.extra_template_basedirs:
path = os.path.join(base_dir, template_name)
if not prune or os.path.exists(path):
paths.append(path)
for root_dir in root_dirs:
base_dir = os.path.join(root_dir, 'nbconvert', 'templates')
path = os.path.join(base_dir, template_name)
if not prune or os.path.exists(path):
paths.append(path)
for root_dir in root_dirs:
# we include root_dir for when we want to be very explicit, e.g.
# {% extends 'nbconvert/templates/classic/base.html' %}
paths.append(root_dir)
# we include base_dir for when we want to be explicit, but less than root_dir, e.g.
# {% extends 'classic/base.html' %}
base_dir = os.path.join(root_dir, 'nbconvert', 'templates')
paths.append(base_dir)
compatibility_dir = os.path.join(root_dir, 'nbconvert', 'templates', 'compatibility')
paths.append(compatibility_dir)
additional_paths = self.template_data_paths
for path in additional_paths:
try:
ensure_dir_exists(path, mode=0o700)
except OSError:
pass
return paths + self.extra_template_paths + additional_paths
def write_connection_file(self):
runtime_dir = jupyter_runtime_dir()
ensure_dir_exists(runtime_dir)
fname = os.path.join(runtime_dir, 'kernelnanny-%s.json' % os.getpid())
with open(fname, 'w') as f:
f.write(json.dumps(self.connection_info, indent=2))
set_sticky_bit(fname)
return fname
def record_connection_info(self, conn_info):
log.info("Connection info: %s", conn_info)
runtime_dir = jupyter_runtime_dir()
ensure_dir_exists(runtime_dir)
fname = os.path.join(runtime_dir, 'kernel-%s.json' % uuid4())
# Only ever write this file as user read/writeable
# This would otherwise introduce a vulnerability as a file has secrets
# which would let others execute arbitrarily code as you
with secure_write(fname) as f:
f.write(json.dumps(conn_info, indent=2))
log.info("To connect a client: --existing %s", os.path.basename(fname))
return fname
def checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip('/')
parent, name = ('/' + path).rsplit('/', 1)
parent = parent.strip('/')
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=ext,
)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
# iOS, move checkpoint directory to ~/Documents if local dir is not writeable
import sys
if (sys.platform == 'darwin' and os.uname().machine.startswith('iP')):
try:
ensure_dir_exists(cp_dir)
if not os.access(cp_dir, os.X_OK | os.W_OK):
cp_dir = os.path.join(os.path.expanduser('~'), 'Documents',
self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
except (OSError, IOError) as e:
cp_dir = os.path.join(os.path.expanduser('~'), 'Documents',
self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
else:
with self.perm_to_403():
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip("/")
parent, name = ("/" + path).rsplit("/", 1)
parent = parent.strip("/")
basename, ext = os.path.splitext(name)
filename = "{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=ext,
)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def make_connection_file(self):
"""Generates a JSON config file, including the selection of random ports.
"""
runtime_dir = jupyter_runtime_dir()
ensure_dir_exists(runtime_dir)
fname = os.path.join(runtime_dir, 'kernel-%s.json' % new_key())
cfg = self.make_ports()
cfg['ip'] = self.ip
cfg['key'] = new_key()
cfg['transport'] = self.transport
cfg['signature_scheme'] = 'hmac-sha256'
with open(fname, 'w') as f:
f.write(json.dumps(cfg, indent=2))
set_sticky_bit(fname)
return fname, cfg
def make_connection_file(self):
"""Generates a JSON config file, including the selection of random ports.
"""
runtime_dir = jupyter_runtime_dir()
ensure_dir_exists(runtime_dir)
fname = os.path.join(runtime_dir, 'kernel-%s.json' % new_key())
cfg = self.make_ports()
cfg['ip'] = self.ip
cfg['key'] = new_key()
cfg['transport'] = self.transport
cfg['signature_scheme'] = 'hmac-sha256'
# Only ever write this file as user read/writeable
# This would otherwise introduce a vulnerability as a file has secrets
# which would let others execute arbitrarily code as you
with secure_write(fname) as f:
f.write(json.dumps(cfg, indent=2))
set_sticky_bit(fname)
return fname, cfg
def _create_environment(self):
"""
Create the Jinja templating environment.
"""
here = os.path.dirname(os.path.realpath(__file__))
additional_paths = self.template_data_paths
for path in additional_paths:
try:
ensure_dir_exists(path, mode=0o700)
except OSError:
pass
paths = self.template_path + \
additional_paths + \
[os.path.join(here, self.default_template_path),
os.path.join(here, self.template_skeleton_path)]
loaders = self.extra_loaders + [
ExtensionTolerantLoader(FileSystemLoader(paths),
self.template_extension),
DictLoader({self._raw_template_key: self.raw_template})
]
environment = Environment(loader=ChoiceLoader(loaders),
extensions=JINJA_EXTENSIONS)
environment.globals['uuid4'] = uuid.uuid4
# Add default filters to the Jinja2 environment
for key, value in self.default_filters():
self._register_filter(environment, key, value)
# Load user filters. Overwrite existing filters if need be.
if self.filters:
for key, user_filter in self.filters.items():
self._register_filter(environment, key, user_filter)
return environment
def install_nbextension(path,
overwrite=False,
symlink=False,
user=False,
prefix=None,
nbextensions_dir=None,
destination=None,
verbose=DEPRECATED_ARGUMENT,
logger=None,
sys_prefix=False):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
path : path to file, directory, zip or tarball archive, or URL to install
By default, the file will be installed with its base name, so '/path/to/foo'
will install to 'nbextensions/foo'. See the destination argument below to change this.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to ``<prefix>/share/jupyter/nbextensions``
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
destination : str [optional]
name the nbextension is installed to. For example, if destination is 'foo', then
the source file will be installed to 'nbextensions/foo', regardless of the source name.
This cannot be specified if an archive is given as the source.
logger : Jupyter logger [optional]
Logger instance to use
"""
if verbose != DEPRECATED_ARGUMENT:
import warnings
warnings.warn(
"`install_nbextension`'s `verbose` parameter is deprecated, it will have no effects and will be removed in Notebook 5.0",
DeprecationWarning)
# the actual path to which we eventually installed
full_dest = None
nbext = _get_nbextension_dir(user=user,
sys_prefix=sys_prefix,
prefix=prefix,
nbextensions_dir=nbextensions_dir)
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
# forcing symlink parameter to False if os.symlink does not exist (e.g., on Windows machines running python 2)
if not hasattr(os, 'symlink'):
symlink = False
if isinstance(path, (list, tuple)):
raise TypeError(
"path must be a string pointing to a single extension to install; call this function multiple times to install multiple extensions"
)
path = cast_unicode_py2(path)
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if logger:
logger.info("Downloading: %s -> %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
full_dest = install_nbextension(local_path,
overwrite=overwrite,
symlink=symlink,
nbextensions_dir=nbext,
destination=destination,
logger=logger)
elif path.endswith('.zip') or _safe_is_tarfile(path):
if symlink:
raise ValueError("Cannot symlink from archives")
if destination:
raise ValueError("Cannot give destination for archives")
if logger:
logger.info("Extracting: %s -> %s" % (path, nbext))
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
archive.extractall(nbext)
archive.close()
# TODO: what to do here
full_dest = None
else:
if not destination:
destination = basename(normpath(path))
destination = cast_unicode_py2(destination)
full_dest = normpath(pjoin(nbext, destination))
if overwrite and os.path.lexists(full_dest):
if logger:
logger.info("Removing: %s" % full_dest)
if os.path.isdir(full_dest) and not os.path.islink(full_dest):
shutil.rmtree(full_dest)
else:
os.remove(full_dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(full_dest):
if logger:
logger.info("Symlinking: %s -> %s" % (full_dest, path))
os.symlink(path, full_dest)
elif os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(full_dest, parent[len(path):])
if not os.path.exists(dest_dir):
if logger:
logger.info("Making directory: %s" % dest_dir)
os.makedirs(dest_dir)
for file_name in files:
src = pjoin(parent, file_name)
dest_file = pjoin(dest_dir, file_name)
_maybe_copy(src, dest_file, logger=logger)
else:
src = path
_maybe_copy(src, full_dest, logger=logger)
return full_dest
def touch(path, content=''):
ensure_dir_exists(os.path.dirname(path))
with open(path, 'w') as f:
f.write(content)
def touch(path, content=''):
ensure_dir_exists(os.path.dirname(path))
with open(path, 'w') as f:
f.write(content)
def develop_labextension(path,
symlink=True,
overwrite=False,
user=False,
labextensions_dir=None,
destination=None,
logger=None,
sys_prefix=False):
"""Install a dynamic extension for JupyterLab
Stages files and/or directories into the labextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
path : path to file, directory, zip or tarball archive, or URL to install
By default, the file will be installed with its base name, so '/path/to/foo'
will install to 'labextensions/foo'. See the destination argument below to change this.
Archives (zip or tarballs) will be extracted into the labextensions directory.
user : bool [default: False]
Whether to install to the user's labextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/labextensions).
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: True]
If True, create a symlink in labextensions, rather than copying files.
Windows support for symlinks requires a permission bit which only admin users
have by default, so don't rely on it.
labextensions_dir : str [optional]
Specify absolute path of labextensions directory explicitly.
destination : str [optional]
name the labextension is installed to. For example, if destination is 'foo', then
the source file will be installed to 'labextensions/foo', regardless of the source name.
logger : Jupyter logger [optional]
Logger instance to use
"""
# the actual path to which we eventually installed
full_dest = None
labext = _get_labextension_dir(user=user,
sys_prefix=sys_prefix,
labextensions_dir=labextensions_dir)
# make sure labextensions dir exists
ensure_dir_exists(labext)
if isinstance(path, (list, tuple)):
raise TypeError(
"path must be a string pointing to a single extension to install; call this function multiple times to install multiple extensions"
)
path = cast_unicode_py2(path)
if not destination:
destination = basename(normpath(path))
destination = cast_unicode_py2(destination)
full_dest = normpath(pjoin(labext, destination))
if overwrite and os.path.lexists(full_dest):
if logger:
logger.info("Removing: %s" % full_dest)
if os.path.isdir(full_dest) and not os.path.islink(full_dest):
shutil.rmtree(full_dest)
else:
os.remove(full_dest)
# Make sure the parent directory exists
os.makedirs(os.path.dirname(full_dest), exist_ok=True)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(full_dest):
if logger:
logger.info("Symlinking: %s -> %s" % (full_dest, path))
os.symlink(path, full_dest)
elif not os.path.islink(full_dest):
raise ValueError("%s exists and is not a symlink" % path)
elif os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(full_dest, parent[len(path):])
if not os.path.exists(dest_dir):
if logger:
logger.info("Making directory: %s" % dest_dir)
os.makedirs(dest_dir)
for file_name in files:
src = pjoin(parent, file_name)
dest_file = pjoin(dest_dir, file_name)
_maybe_copy(src, dest_file, logger=logger)
else:
src = path
_maybe_copy(src, full_dest, logger=logger)
return full_dest
def install_nbextension(path, overwrite=False, symlink=False,
user=False, prefix=None, nbextensions_dir=None,
destination=None, verbose=DEPRECATED_ARGUMENT,
logger=None, sys_prefix=False
):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
path : path to file, directory, zip or tarball archive, or URL to install
By default, the file will be installed with its base name, so '/path/to/foo'
will install to 'nbextensions/foo'. See the destination argument below to change this.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to ``<prefix>/share/jupyter/nbextensions``
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
destination : str [optional]
name the nbextension is installed to. For example, if destination is 'foo', then
the source file will be installed to 'nbextensions/foo', regardless of the source name.
This cannot be specified if an archive is given as the source.
logger : Jupyter logger [optional]
Logger instance to use
"""
if verbose != DEPRECATED_ARGUMENT:
import warnings
warnings.warn("`install_nbextension`'s `verbose` parameter is deprecated, it will have no effects and will be removed in Notebook 5.0", DeprecationWarning)
# the actual path to which we eventually installed
full_dest = None
nbext = _get_nbextension_dir(user=user, sys_prefix=sys_prefix, prefix=prefix, nbextensions_dir=nbextensions_dir)
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
# forcing symlink parameter to False if os.symlink does not exist (e.g., on Windows machines running python 2)
if not hasattr(os, 'symlink'):
symlink = False
if isinstance(path, (list, tuple)):
raise TypeError("path must be a string pointing to a single extension to install; call this function multiple times to install multiple extensions")
path = cast_unicode_py2(path)
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if logger:
logger.info("Downloading: %s -> %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
full_dest = install_nbextension(local_path, overwrite=overwrite, symlink=symlink,
nbextensions_dir=nbext, destination=destination, logger=logger)
elif path.endswith('.zip') or _safe_is_tarfile(path):
if symlink:
raise ValueError("Cannot symlink from archives")
if destination:
raise ValueError("Cannot give destination for archives")
if logger:
logger.info("Extracting: %s -> %s" % (path, nbext))
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
archive.extractall(nbext)
archive.close()
# TODO: what to do here
full_dest = None
else:
if not destination:
destination = basename(normpath(path))
destination = cast_unicode_py2(destination)
full_dest = normpath(pjoin(nbext, destination))
if overwrite and os.path.lexists(full_dest):
if logger:
logger.info("Removing: %s" % full_dest)
if os.path.isdir(full_dest) and not os.path.islink(full_dest):
shutil.rmtree(full_dest)
else:
os.remove(full_dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(full_dest):
if logger:
logger.info("Symlinking: %s -> %s" % (full_dest, path))
os.symlink(path, full_dest)
elif os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(full_dest, parent[len(path):])
if not os.path.exists(dest_dir):
if logger:
logger.info("Making directory: %s" % dest_dir)
os.makedirs(dest_dir)
for file_name in files:
src = pjoin(parent, file_name)
dest_file = pjoin(dest_dir, file_name)
_maybe_copy(src, dest_file, logger=logger)
else:
src = path
_maybe_copy(src, full_dest, logger=logger)
return full_dest