def test_data_dir_linux():
with linux, no_xdg:
data = jupyter_data_dir()
assert data == realpath('~/.local/share/jupyter')
with linux, xdg:
data = jupyter_data_dir()
assert data == pjoin(xdg_env['XDG_DATA_HOME'], 'jupyter')
def test_data_dir_windows():
with windows, appdata:
data = jupyter_data_dir()
assert data == pjoin('appdata', 'jupyter')
with windows, appdata, xdg:
# windows should ignore xdg
data = jupyter_data_dir()
assert data == pjoin('appdata', 'jupyter')
def test_data_dir_darwin():
with darwin:
data = jupyter_data_dir()
assert data == realpath('~/Library/Jupyter')
with darwin, xdg:
# darwin should ignore xdg
data = jupyter_data_dir()
assert data == realpath('~/Library/Jupyter')
def _nbextension_dirs():
"""The possible locations of nbextensions."""
return [
os.path.join(jupyter_data_dir(), u'nbextensions'),
os.path.join(ENV_JUPYTER_PATH[0], u'nbextensions'),
os.path.join(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
]
def _get_nbextension_dir(user=False, sys_prefix=False, prefix=None, nbextensions_dir=None):
"""Return the nbextension directory specified
Parameters
----------
user : bool [default: False]
Get the user's .jupyter/nbextensions directory
sys_prefix : bool [default: False]
Get sys.prefix, i.e. ~/.envs/my-env/share/jupyter/nbextensions
prefix : str [optional]
Get custom prefix
nbextensions_dir : str [optional]
Get what you put in
"""
if sum(map(bool, [user, prefix, nbextensions_dir, sys_prefix])) > 1:
raise ArgumentConflict("cannot specify more than one of user, sys_prefix, prefix, or nbextensions_dir")
if user:
nbext = pjoin(jupyter_data_dir(), u'nbextensions')
elif sys_prefix:
nbext = pjoin(ENV_JUPYTER_PATH[0], u'nbextensions')
elif prefix:
nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
return nbext
def _init_index(self, reset=False):
index_path = os.path.join(jupyter_data_dir(), "index")
# clear out old index if requested
if reset:
shutil.rmtree(index_path, True)
# make sure there's a path to store the index data
if not os.path.exists(index_path):
os.makedirs(index_path)
if not exists_in(index_path):
# create an index with the current schema
analyzer = ChineseAnalyzer()
schema = Schema(
basename=TEXT(stored=True, field_boost=5.0, analyzer=analyzer),
dirname=ID(stored=True, analyzer=analyzer),
path=ID(stored=True, unique=True, analyzer=analyzer),
content=TEXT(stored=False, analyzer=analyzer),
time=STORED,
)
self.ix = create_in(index_path, schema)
else:
# open the existing index
self.ix = open_dir(index_path)
# build a query parser based on the current schema
self.query_parser = MultifieldParser(["content", "basename", "dirname"], self.ix.schema)
def _uninstall_pre_files(logger=None):
"""
Remove any files recorded from a previous installation.
Rather than actually deleting, this function copies everything to a
temporary directory without explicit cleanup, in case a user wants to try
manual recovery at some point.
The OS can then handle actual removal from the temp directory as and when
it chooses to.
"""
data_dir = jupyter_data_dir()
bom_pref = 'ipython-contrib-IPython-notebook-extensions-'
bom_path = os.path.join(data_dir, bom_pref + 'installed_files.txt')
if not os.path.exists(bom_path):
if logger:
logger.info('- No list of previously-installed files at {}'.format(
bom_path))
return
elif logger:
logger.info(
'- Removing previously-installed files listed in {}'.format(
bom_path))
deleted_to = tempfile.mkdtemp(prefix=bom_pref)
if logger:
logger.info(
'-- Files will be copied to the temp directory {}'.format(
deleted_to))
with open(bom_path, 'r') as bom_file:
for src in bom_file.readlines():
src = src.rstrip('\n').rstrip('\r')
if os.path.exists(src):
if logger:
logger.info(' ' + src)
dest = os.path.join(
deleted_to, os.path.relpath(src, data_dir))
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.move(src, dest)
# remove empty directories
allowed_errnos = (errno.ENOTDIR, errno.ENOTEMPTY, errno.ENOENT)
while len(src) > len(data_dir):
src = os.path.dirname(src)
try:
os.rmdir(src)
except OSError as ex:
if ex.errno not in allowed_errnos:
raise
break
else:
if logger:
logger.info(' ' + src)
os.remove(bom_path)
return deleted_to
def test_install_user():
tmp = tempfile.mkdtemp()
with mock.patch.dict(os.environ, {'HOME': tmp}):
install(user=True)
data_dir = jupyter_data_dir()
assert_is_spec(os.path.join(data_dir, 'kernels', KERNEL_NAME))
def bundle_declarative_widgets(output_path, notebook_file):
'''
Adds frontend bower components dependencies into the bundle for the dashboard
application. Creates the following directories under output_path:
static/urth_widgets: Stores the js for urth_widgets which will be loaded in
the frontend of the dashboard
static/urth_components: The directory for all of the bower components of the
dashboard.
NOTE: This function is too specific to urth widgets. In the
future we should investigate ways to make this more generic.
:param output_path: The output path of the dashboard being assembled
:param notebook_file: The absolute path to the notebook file being packaged
'''
widgets_dir = pjoin(jupyter_data_dir(), 'nbextensions/urth_widgets')
if not os.path.isdir(widgets_dir):
return
# Check if any of the cells contain widgets, if not we do not to copy the bower_components
notebook = nbformat.read(notebook_file, 4)
# Using find instead of a regex to help future-proof changes that might be
# to how user's will use urth-core-import
# (i.e. <link is=urth-core-import> vs. <urth-core-import>)
any_cells_with_widgets = any(cell.get('source').find('urth-core-import') != -1 for cell in notebook.cells)
if not any_cells_with_widgets:
return
# Directory of declarative widgets extension
widgets_dir = pjoin(jupyter_data_dir(), 'nbextensions/urth_widgets')
# Root of declarative widgets within a dashboard app
output_widgets_dir = pjoin(output_path, 'static/urth_widgets/')
# JavaScript entry point for widgets in dashboard app
output_js_dir = pjoin(output_widgets_dir, 'js')
# Web referenceable path from which all urth widget components will be served
output_components_dir = pjoin(output_path, 'static/urth_components/')
# Copy declarative widgets js and installed bower components into the app
# under output directory
widgets_js_dir = pjoin(widgets_dir, 'js')
shutil.copytree(widgets_js_dir, output_js_dir)
# Install the bower components into the output components directory
shutil.copytree(pjoin(widgets_dir, 'bower_components'), output_components_dir)
def _install_test_kernel(self):
kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'signaltest')
os.makedirs(kernel_dir)
with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f:
f.write(json.dumps({
'argv': [sys.executable,
'-m', 'jupyter_client.tests.signalkernel',
'-f', '{connection_file}'],
'display_name': "Signal Test Kernel",
}))
def _nbextension_dirs():
"""The possible locations of nbextensions.
Returns a list of known base extension locations
"""
return [
pjoin(jupyter_data_dir(), u'nbextensions'),
pjoin(ENV_JUPYTER_PATH[0], u'nbextensions'),
pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
]
def test_paths():
output = get_jupyter_output("--paths")
for d in (jupyter_config_dir(), jupyter_data_dir(), jupyter_runtime_dir()):
assert d in output
for key in ("config", "data", "runtime"):
assert ("%s:" % key) in output
for path in (jupyter_config_path(), jupyter_path()):
for d in path:
assert d in output
def nbextension_ordered_paths():
import jupyter_core.paths as jupypaths
jupyter_searchpath = jupypaths.jupyter_path()
paths = [('user', jupypaths.jupyter_data_dir()),
('environment', jupypaths.ENV_JUPYTER_PATH[0]),
('system', jupypaths.SYSTEM_JUPYTER_PATH[0])]
paths.sort(key=lambda x: jupyter_searchpath.index(x[1]))
return collections.OrderedDict(paths)
def test_paths():
output = get_jupyter_output('--paths')
for d in (jupyter_config_dir(), jupyter_data_dir(), jupyter_runtime_dir()):
assert d in output
for key in ('config', 'data', 'runtime'):
assert ('%s:' % key) in output
for path in (jupyter_config_path(), jupyter_path()):
for d in path:
assert d in output
def init(self):
# Get connection info from the PyLL JSON file
CONN_FILE = jupyter_data_dir() + "/runtime/" + "virtuoso-pyll.json"
with open(CONN_FILE, "r") as COF:
self.host, self.port = json.load(COF)
# Connection info will come from a JSON file generated by the dfII/PyLL
# server. So, read JSON to figure out the connection info.
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REQ)
self.socket.connect("tcp://%s:%d" % (self.host, self.port))
def task_copy_template():
global template
template_dir = Path(jupyter_data_dir()) / 'templates'
yield {'name': 'copy_template_to_jupyter_dir',
'actions': ['cp {0} {1}'.format(template, template_dir)],
'targets': [str(template_dir / template)],
'file_dep': [str(template)],
'clean': True}
def installNbExtensions(p, update=0,verbose=0, debug = False):
# Install notebook extensions
from jupyter_core.paths import jupyter_data_dir
import os
import sys
import shutil
import IPython
import notebook
if IPython.__version__[0] < '4':
print("IPython version 4.x is required")
exit(1)
print("Installing Jupyter notebook extensions.")
#
# 1. Get the local configuration file path
#
data_dir = os.getenv('PREFIX', None)
if data_dir == None:
data_dir = jupyter_data_dir()
else:
data_dir = os.path.join(data_dir, 'share/jupyter')
print("Extensions and templates path: %s" % data_dir)
if os.path.exists(data_dir) is False:
os.mkdir(data_dir)
if debug is True: print("Creating directory %s" % data_dir)
#
# 2. Install files
#
# copy extensions to IPython extensions directory
src = os.path.join(p,'extensions')
destination = os.path.join(data_dir, 'extensions')
if debug is True: print("Install Python extensions to %s" % destination)
recursive_copy(src, destination,update=update)
# Install templates
src = os.path.join(p,'templates')
destination = os.path.join(data_dir, 'templates')
if debug is True: print("Install templates to %s" % destination)
recursive_copy(src, destination,update=update)
# Install nbextensions
src = os.path.join(p,'nbextensions')
destination = os.path.join(data_dir, 'nbextensions')
if debug is True: print("Install notebook extensions to %s" % destination)
recursive_copy(src, destination,update=update)
def reset_default():
""" remove custom.css import"""
from jupyter_core.paths import jupyter_data_dir
jnb_cached = os.path.join(jupyter_data_dir(), 'nbextensions')
paths = [jupyter_custom, jnb_cached]
for fpath in paths:
custom = '{0}/{1}.css'.format(fpath, 'custom')
try:
os.remove(custom)
print("Reset default theme in: {0}".format(fpath))
except Exception:
print("Already set to default theme in {0}".format(fpath))
def _install_test_kernel(self):
kernel_dir = pjoin(paths.jupyter_data_dir(), "kernels", "signaltest")
os.makedirs(kernel_dir)
with open(pjoin(kernel_dir, "kernel.json"), "w") as f:
f.write(
json.dumps(
{
"argv": [sys.executable, "-m", "jupyter_client.tests.signalkernel", "-f", "{connection_file}"],
"display_name": "Signal Test Kernel",
}
)
)
def setUp(self):
self.env_patch = test_env()
self.env_patch.start()
self.sample_kernel_dir = self._install_sample_kernel(
pjoin(paths.jupyter_data_dir(), 'kernels'))
self.ksm = kernelspec.KernelSpecManager()
td2 = TemporaryDirectory()
self.addCleanup(td2.cleanup)
self.installable_kernel = td2.name
with open(pjoin(self.installable_kernel, 'kernel.json'), 'w') as f:
json.dump(sample_kernel_json, f)
def _install_test_kernel(self):
kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'signaltest')
os.makedirs(kernel_dir)
with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f:
f.write(
json.dumps({
'argv': [
sys.executable, '-m',
'jupyter_client.tests.signalkernel', '-f',
'{connection_file}'
],
'display_name':
"Signal Test Kernel",
}))
def run(self):
src_dir = os.path.join(repo_root, 'odps', 'static', 'ui', 'target')
dest_dir = os.path.join(jupyter_data_dir(), 'nbextensions', 'pyodps')
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
recursive_overwrite(src_dir, dest_dir)
try:
from notebook.nbextensions import enable_nbextension
except ImportError:
return
enable_nbextension('notebook', 'pyodps/main')
def _get_nbext_dir(nbextensions_dir=None, user=False, prefix=None):
"""Return the nbextension directory specified"""
if sum(map(bool, [user, prefix, nbextensions_dir])) > 1:
raise ArgumentConflict("Cannot specify more than one of user, prefix, or nbextensions_dir.")
if user:
nbext = pjoin(jupyter_data_dir(), u'nbextensions')
else:
if prefix:
nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
return nbext
async def test_bad_kernelspec_pending(self):
km = self._get_pending_kernels_km()
install_kernel(
os.path.join(paths.jupyter_data_dir(), "kernels"),
argv=["non_existent_executable"],
name="bad",
)
kernel_id = await ensure_future(
km.start_kernel(kernel_name="bad", stdout=PIPE, stderr=PIPE))
with pytest.raises(FileNotFoundError):
await km.get_kernel(kernel_id).ready
assert kernel_id in km.list_kernel_ids()
await ensure_future(km.shutdown_kernel(kernel_id))
assert kernel_id not in km.list_kernel_ids()
def run(self):
src_dir = os.path.join(repo_root, 'odps', 'static', 'ui', 'target')
dest_dir = os.path.join(jupyter_data_dir(), 'nbextensions', 'pyodps')
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
recursive_overwrite(src_dir, dest_dir)
try:
from notebook.nbextensions import enable_nbextension
except ImportError:
return
enable_nbextension('notebook', 'pyodps/main')
def _get_nbext_dir(nbextensions_dir=None, user=False, prefix=None):
"""Return the nbextension directory specified"""
if sum(map(bool, [user, prefix, nbextensions_dir])) > 1:
raise ArgumentConflict("Cannot specify more than one of user, prefix, or nbextensions_dir.")
if user:
nbext = pjoin(jupyter_data_dir(), u'nbextensions')
else:
if prefix:
nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
return nbext
def get(self):
jupyterdir = jupyter_data_dir()
nbextensions = (get_nbext_dir(),
os.path.join(jupyterdir, 'nbextensions'))
exclude = ['mathjax']
yaml_list = []
# Traverse through nbextension subdirectories to find all yaml files
for root, dirs, files in chain.from_iterable(
os.walk(root) for root in nbextensions):
dirs[:] = [d for d in dirs if d not in exclude]
for f in files:
if f.endswith('.yaml'):
yaml_list.append([root, f])
# Build a list of extensions from YAML file description
# containing at least the following entries:
# Type - identifier
# Name - unique name of the extension
# Description - short explanation of the extension
# Main - main file that is loaded, typically 'main.js'
#
extension_list = []
for y in yaml_list:
stream = open(os.path.join(y[0], y[1]), 'r')
extension = yaml.load(stream)
if all(k in extension for k in ('Type', 'Compatibility', 'Name',
'Main', 'Description')):
if not extension['Type'].strip().startswith(
'IPython Notebook Extension'):
continue
if not extension['Compatibility'].strip().startswith('3.'):
continue
# generate URL to extension
idx = y[0].find('nbextensions')
url = y[0][idx::].replace('\\', '/')
extension['url'] = url
# replace single quote with HTML representation
for key in extension:
if isinstance(extension[key], str):
extension[key] = extension[key].replace("'", "'")
extension_list.append(extension)
self.log.info("Found extension %s" % extension['Name'])
stream.close()
json_list = json.dumps(extension_list)
self.write(
self.render_template(
'nbextensions.html',
base_url=self.base_url,
extension_list=json_list,
page_title="Notebook Extension Configuration"))
def install_echarts_if_needed():
"""
Copy all echarts javascripts to jupyter_data_dir
"""
import shutil
from jupyter_core.paths import jupyter_data_dir
nbextension_path = os.path.join(jupyter_data_dir(), NBEXT_NAME)
pyecharts_signature = os.path.join(nbextension_path,
'.pyecharts.signature')
if os.path.exists(pyecharts_signature) is False:
js_folder = template.get_resource_dir(os.path.join('templates', 'js'))
for js_file in os.listdir(js_folder):
shutil.copy(os.path.join(js_folder, js_file),
os.path.join(nbextension_path, js_file))
__create_pyecharts_signature(pyecharts_signature)
def reset_default():
""" remove custom.css import"""
from jupyter_core.paths import jupyter_data_dir
jnb_cached = os.path.join(jupyter_data_dir(), 'nbextensions')
paths = [INSTALL_JPATH, jnb_cached]
for fpath in paths:
old = '%s/%s.css' % (fpath, 'custom')
old_save = '%s/%s.css' % (fpath, 'custom_old')
try:
shutil.copy(old, old_save)
os.remove(old)
print("Reset default theme here: %s" % fpath)
except Exception:
print("Already set to default theme in %s" % fpath)
pass
def reset_default():
""" remove custom.css import"""
from jupyter_core.paths import jupyter_data_dir
jnb_cached = os.path.join(jupyter_data_dir(), 'nbextensions')
paths = [INSTALL_JPATH, jnb_cached]
for fpath in paths:
old = '%s/%s.css' % (fpath, 'custom')
old_save = '%s/%s.css' % (fpath, 'custom_old')
try:
shutil.copy(old, old_save)
os.remove(old)
print("Reset default theme here: %s" % fpath)
except Exception:
print("Already set to default theme in %s" % fpath)
pass
def get(self):
jupyterdir = jupyter_data_dir()
nbextensions = (get_nbext_dir(), os.path.join(jupyterdir,'nbextensions'))
exclude = [ 'mathjax' ]
yaml_list = []
# Traverse through nbextension subdirectories to find all yaml files
for root, dirs, files in chain.from_iterable(os.walk(root) for root in nbextensions):
dirs[:] = [d for d in dirs if d not in exclude]
for f in files:
if f.endswith('.yaml'):
yaml_list.append([ root, f] )
# Build a list of extensions from YAML file description
# containing at least the following entries:
# Type - identifier
# Name - unique name of the extension
# Description - short explanation of the extension
# Main - main file that is loaded, typically 'main.js'
#
extension_list = []
for y in yaml_list:
stream = open(os.path.join(y[0],y[1]), 'r')
extension = yaml.load(stream)
if all (k in extension for k in ('Type', 'Compatibility', 'Name', 'Main', 'Description')):
if not extension['Type'].strip().startswith('IPython Notebook Extension'):
continue
if not extension['Compatibility'].strip().startswith('3.'):
continue
# generate URL to extension
idx=y[0].find('nbextensions')
url = y[0][idx::].replace('\\', '/')
extension['url'] = url
# replace single quote with HTML representation
for key in extension:
if isinstance(extension[key], str):
extension[key] = extension[key].replace("'","'")
extension_list.append(extension)
self.log.info("Found extension %s" % extension['Name'])
stream.close()
json_list = json.dumps(extension_list)
self.write(self.render_template('nbextensions.html',
base_url = self.base_url,
extension_list = json_list,
page_title="Notebook Extension Configuration"
)
)
def _get_nbextension_dir(user=False,
sys_prefix=False,
prefix=None,
nbextensions_dir=None,
root=None):
"""Return the nbextension directory specified
Parameters
----------
user : bool [default: False]
Get the user's .jupyter/nbextensions directory
sys_prefix : bool [default: False]
Get sys.prefix, i.e. ~/.envs/my-env/share/jupyter/nbextensions
prefix : str [optional]
Get custom prefix
nbextensions_dir : str [optional]
Get what you put ins
root : str [optional]
Get custom root directory
"""
conflicting = [
('user', user),
('prefix', prefix),
('nbextensions_dir', nbextensions_dir),
('sys_prefix', sys_prefix),
]
conflicting_set = ['{}={!r}'.format(n, v) for n, v in conflicting if v]
if len(conflicting_set) > 1:
raise ArgumentConflict(
"cannot specify more than one of user, sys_prefix, prefix, or nbextensions_dir, but got: {}"
.format(', '.join(conflicting_set)))
if user:
nbext = pjoin(jupyter_data_dir(), u'nbextensions')
elif sys_prefix:
nbext = pjoin(ENV_JUPYTER_PATH[0], u'nbextensions')
elif prefix:
nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = pjoin(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
if root:
nbext = pjoin(root, nbext)
return nbext
def main():
parser = argparse.ArgumentParser()
aa = parser.add_argument
aa('-n', '--name', nargs=1, required=1, help='Display name of kernel')
aa('-i',
'--image',
nargs=1,
required=1,
help='Docker image used for creating docker containers to proxy into')
aa('--kernel-dir-name',
nargs=1,
default=None,
help='Optionally override the kernel directory name. Not a full path.')
aa('other_args',
nargs='*',
default=[],
help='Additional arguments to pass to kernel')
args = parser.parse_args()
kernel_name = args.name[0]
kernel_dir = os.path.join(jupyter_data_dir(), 'kernels')
if args.kernel_dir_name:
kernel_dir = os.path.join(kernel_dir, args.kernel_dir_name)
else:
kernel_dir = os.path.join(kernel_dir, kernel_name.replace('/', ''))
if not os.path.exists(kernel_dir):
os.makedirs(kernel_dir)
else:
raise Exception('Kernel directory exists')
kernel = {
'display_name':
kernel_name,
'argv': [
sys.executable, '-m', 'docker_proxy_kernel', '-f',
'{connection_file}', '--image', args.image[0]
] + args.other_args,
'env': {},
'language':
'python'
}
with open(os.path.join(kernel_dir, 'kernel.json'), 'w') as fid:
json.dump(kernel, fid, indent=2)
def _get_nbextension_dir(user=False, sys_prefix=False, prefix=None,
nbextensions_dir=None):
"""Return the nbextension directory specified."""
if sum(map(bool, [user, prefix, nbextensions_dir, sys_prefix])) > 1:
raise ArgumentConflict(
"cannot specify more than one of user, sys_prefix, prefix, "
"or nbextensions_dir")
if user:
nbext = os.path.join(jupyter_data_dir(), u'nbextensions')
elif sys_prefix:
nbext = os.path.join(ENV_JUPYTER_PATH[0], u'nbextensions')
elif prefix:
nbext = os.path.join(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = os.path.join(SYSTEM_JUPYTER_PATH[0], 'nbextensions')
return nbext
def bundle_web_static(output_path):
'''
Copies all the static web assets needed for the dashboard to run stadalone
to the static/ subdir of the output directory.
:param output_path: The output path of the dashboard being assembled
'''
# Copy static assets for Thebe
shutil.copytree(STATICS_PATH, pjoin(output_path, 'static'))
# Select specific files from bower_components, not whole git clones
components = [
'bower_components/gridstack/dist/gridstack.min.css',
'bower_components/gridstack/dist/gridstack.min.js',
'bower_components/gridstack/dist/gridstack.min.map',
'bower_components/jquery/dist/jquery.min.js',
'bower_components/jquery/dist/jquery.min.map',
'bower_components/jquery-ui/themes/smoothness/jquery-ui.min.css',
'bower_components/lodash/lodash.min.js',
'bower_components/requirejs/require.js'
]
# Select only dashboard common files
component_dirs = [
'dashboard-common',
'bower_components/jquery-ui/themes/smoothness/images'
]
# Paths for dashboard extension source and destination in local app folder
src_components = pjoin(jupyter_data_dir(),
'nbextensions/urth_dash_js/notebook')
dest_components = pjoin(output_path, 'static')
# Copy individual files, making directories as we go
for component in components:
dest_file = pjoin(dest_components, component)
dest_dir = os.path.dirname(dest_file)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy(pjoin(src_components, component), dest_file)
# Copy entire directories
for comp_dir in component_dirs:
shutil.copytree(pjoin(src_components, comp_dir),
pjoin(dest_components, comp_dir))
def add_urth_widgets(output_path, notebook_file):
'''
Adds frontend bower components dependencies into the bundle for the dashboard
application. Creates the following directories under output_path:
static/urth_widgets: Stores the js for urth_widgets which will be loaded in
the frontend of the dashboard
static/urth_components: The directory for all of the bower components of the
dashboard.
NOTE: This function is too specific to urth widgets. In the
future we should investigate ways to make this more generic.
:param output_path: The output path of the dashboard being assembled
:param notebook_file: The absolute path to the notebook file being packaged
'''
ipython_dir = jupyter_data_dir()
# Root of urth widgets within Jupyter
urth_widgets_dir = os.path.join(ipython_dir, 'nbextensions/urth_widgets')
# JavaScript entry point for widgets in Jupyter
urth_widgets_js_dir = os.path.join(urth_widgets_dir, 'js')
if not os.path.isdir(urth_widgets_dir):
# urth widgets not installed so skip
return
# Check if any of the cells contain widgets, if not we do not to copy the bower_components
notebook = nbformat.read(notebook_file, 4)
any_cells_with_widgets = any(cell_uses_widgets(cell.get('source')) for cell in notebook.cells)
if not any_cells_with_widgets:
return
# Root of urth widgets within a dashboard app
output_urth_widgets_dir = os.path.join(output_path, 'static/urth_widgets/')
# JavaScript entry point for widgets in dashboard app
output_js_dir = os.path.join(output_urth_widgets_dir, 'js')
# Web referenceable path from which all urth widget components will be served
output_urth_components_dir = os.path.join(output_path, 'static/urth_components/')
# Copy urth widgets js and installed bower components into the app under
# static/urth_widgets
shutil.copytree(urth_widgets_js_dir, output_js_dir)
# Install the bower components into the urth_components directory
shutil.copytree(os.path.join(urth_widgets_dir, 'bower_components'), output_urth_components_dir)
def bundle_web_static(output_path):
'''
Copies all the static web assets needed for the dashboard to run stadalone
to the static/ subdir of the output directory.
:param output_path: The output path of the dashboard being assembled
'''
# Copy static assets for Thebe
shutil.copytree(STATICS_PATH, pjoin(output_path, 'static'))
# Select specific files from bower_components, not whole git clones
components = [
'bower_components/gridstack/dist/gridstack.min.css',
'bower_components/gridstack/dist/gridstack.min.js',
'bower_components/gridstack/dist/gridstack.min.map',
'bower_components/jquery/dist/jquery.min.js',
'bower_components/jquery/dist/jquery.min.map',
'bower_components/jquery-ui/themes/smoothness/jquery-ui.min.css',
'bower_components/lodash/lodash.min.js',
'bower_components/requirejs/require.js'
]
# Select only dashboard common files
component_dirs = [
'dashboard-common',
'bower_components/jquery-ui/themes/smoothness/images'
]
# Paths for dashboard extension source and destination in local app folder
src_components = pjoin(jupyter_data_dir(), 'nbextensions/jupyter_dashboards/notebook')
dest_components = pjoin(output_path, 'static')
# Copy individual files, making directories as we go
for component in components:
dest_file = pjoin(dest_components, component)
dest_dir = os.path.dirname(dest_file)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
shutil.copy(pjoin(src_components, component), dest_file)
# Copy entire directories
for comp_dir in component_dirs:
shutil.copytree(pjoin(src_components, comp_dir), pjoin(dest_components, comp_dir))
def install_echarts_if_needed():
"""
Copy all echarts javascripts to jupyter_data_dir
"""
import shutil
from jupyter_core.paths import jupyter_data_dir
nbextension_path = os.path.join(jupyter_data_dir(), NBEXT_NAME)
if os.path.exists(nbextension_path) is False:
os.mkdir(nbextension_path)
pyecharts_signature = os.path.join(
nbextension_path, NBEXT_SIGNATURE)
if os.path.exists(pyecharts_signature) is False:
# site_packages/pyecharts/templates/js
js_folder = template.get_resource_dir(
os.path.join('templates', 'js'))
all_js_files = os.listdir(js_folder)
for js_file in all_js_files:
shutil.copy(os.path.join(js_folder, js_file),
os.path.join(nbextension_path, js_file))
__create_pyecharts_signature(pyecharts_signature, all_js_files)
def get_ipython_dir(profile=None):
"""Returns the base directory of the IPython server
:param profile: The name of the IPython profile
:type profile: str
"""
if not check_ipython():
raise ImportError("Cannot Find IPython Environment")
import IPython
# IPython 4.0+ changes the position of files in the profile
# directory
if V(IPython.__version__) >= V('4.0.0'):
from jupyter_core.paths import jupyter_data_dir
return os.path.join(jupyter_data_dir(), IPYTHON_V4_BASE.strip("/"))
else:
if not profile:
profile = get_profile_name()
return os.path.join(IPython.utils.path.locate_profile(profile),
IPYTHON_V3_BASE.strip("/"))
def _install_kernel(name="signaltest", extra_env=None):
if extra_env is None:
extra_env = {}
kernel_dir = pjoin(paths.jupyter_data_dir(), "kernels", name)
os.makedirs(kernel_dir)
with open(pjoin(kernel_dir, "kernel.json"), "w") as f:
f.write(
json.dumps(
{
"argv": [
sys.executable,
"-m",
"jupyter_client.tests.signalkernel",
"-f",
"{connection_file}",
],
"display_name": "Signal Test Kernel",
"env": {"TEST_VARS": "${TEST_VARS}:test_var_2", **extra_env},
}
)
)
def _get_labextension_dir(user=False,
sys_prefix=False,
prefix=None,
labextensions_dir=None):
"""Return the labextension directory specified
Parameters
----------
user : bool [default: False]
Get the user's .jupyter/labextensions directory
sys_prefix : bool [default: False]
Get sys.prefix, i.e. ~/.envs/my-env/share/jupyter/labextensions
prefix : str [optional]
Get custom prefix
labextensions_dir : str [optional]
Get what you put in
"""
conflicting = [
("user", user),
("prefix", prefix),
("labextensions_dir", labextensions_dir),
("sys_prefix", sys_prefix),
]
conflicting_set = ["{}={!r}".format(n, v) for n, v in conflicting if v]
if len(conflicting_set) > 1:
raise ArgumentConflict(
"cannot specify more than one of user, sys_prefix, prefix, or labextensions_dir, but got: {}"
.format(", ".join(conflicting_set)))
if user:
labext = pjoin(jupyter_data_dir(), "labextensions")
elif sys_prefix:
labext = pjoin(ENV_JUPYTER_PATH[0], "labextensions")
elif prefix:
labext = pjoin(prefix, "share", "jupyter", "labextensions")
elif labextensions_dir:
labext = labextensions_dir
else:
labext = pjoin(SYSTEM_JUPYTER_PATH[0], "labextensions")
return labext
def load_jupyter_server_extension(nb_app):
global logger
global widgets_dir
logger = nb_app.log
logger.info('Loading urth_import server extension.')
# Determine the nbextensions directory and urth_widgets path
ipython_dir = jupyter_data_dir()
web_app = nb_app.web_app
for path in web_app.settings['nbextensions_path']:
if ipython_dir in path:
nbext = path
widgets_dir = os.path.join(nbext, 'urth_widgets/')
# Write out a .bowerrc file to configure bower installs to
# not be interactive and not to prompt for analytics
bowerrc = os.path.join(widgets_dir, '.bowerrc')
if os.access(bowerrc, os.F_OK) is not True:
logger.debug('Writing .bowerrc at {0}'.format(bowerrc))
with open(bowerrc, 'a') as f:
f.write("""{
"analytics": false,
"interactive": false,
"directory": "urth_components"
}""")
# The import handler serves from /urth_import and any requests
# containing /urth_components/ will get served from the actual
# urth_components directory.
import_route_pattern = url_path_join(web_app.settings['base_url'], '/urth_import')
components_route_pattern = url_path_join(web_app.settings['base_url'], '/urth_components/(.*)')
components_path = os.path.join(widgets_dir, 'urth_components/')
# Register the Urth import handler and static file handler.
logger.debug('Adding handlers for {0} and {1}'.format(import_route_pattern, components_route_pattern))
web_app.add_handlers('.*$', [
(import_route_pattern, UrthImportHandler, dict(executor=ThreadPoolExecutor(max_workers=1))),
(components_route_pattern, FileFindHandler, {'path': [components_path]})
])
def _get_nbextension_dir(user=False, sys_prefix=False, prefix=None, nbextensions_dir=None):
"""Return the nbextension directory specified
Parameters
----------
user : bool [default: False]
Get the user's .jupyter/nbextensions directory
sys_prefix : bool [default: False]
Get sys.prefix, i.e. ~/.envs/my-env/share/jupyter/nbextensions
prefix : str [optional]
Get custom prefix
nbextensions_dir : str [optional]
Get what you put in
"""
conflicting = [
("user", user),
("prefix", prefix),
("nbextensions_dir", nbextensions_dir),
("sys_prefix", sys_prefix),
]
conflicting_set = ["{}={!r}".format(n, v) for n, v in conflicting if v]
if len(conflicting_set) > 1:
raise ArgumentConflict(
"cannot specify more than one of user, sys_prefix, prefix, or nbextensions_dir, but got: {}".format(
", ".join(conflicting_set)
)
)
if user:
nbext = pjoin(jupyter_data_dir(), u"nbextensions")
elif sys_prefix:
nbext = pjoin(ENV_JUPYTER_PATH[0], u"nbextensions")
elif prefix:
nbext = pjoin(prefix, "share", "jupyter", "nbextensions")
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = pjoin(SYSTEM_JUPYTER_PATH[0], "nbextensions")
return nbext
def get_ipython_dir(profile=None):
"""Returns the base directory of the IPython server
:param profile: The name of the IPython profile
:type profile: str
"""
if not check_ipython():
raise ImportError("Cannot Find IPython Environment")
import IPython
# IPython 4.0+ changes the position of files in the profile
# directory
if V(IPython.__version__) >= V("4.0.0"):
from jupyter_core.paths import jupyter_data_dir
return os.path.join(jupyter_data_dir(), IPYTHON_V4_BASE.strip("/"))
else:
if not profile:
profile = get_profile_name()
return os.path.join(IPython.utils.path.locate_profile(profile), IPYTHON_V3_BASE.strip("/"))
def install_kernel(self, test_wasm_path):
"""Install the test kernel to Jupyter.
Adapted from https://github.com/jupyter/jupyter_client/blob/284914b/jupyter_client/tests/test_kernelmanager.py#L56
"""
kernel_dir = os.path.join(paths.jupyter_data_dir(), "kernels", TEST_KERNEL_NAME)
os.makedirs(kernel_dir, exist_ok=True)
with open(os.path.join(kernel_dir, "kernel.json"), "w") as f:
f.write(
json.dumps(
{
"argv": [
sys.executable,
"-m",
"wasm_spec_kernel",
"-f",
"{connection_file}",
],
"display_name": "Test Wasm",
"env": {"WASM_INTERPRETER": test_wasm_path},
}
)
)
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import unittest
import shutil
import os
import tempfile
import dashboards_bundlers.local_deploy as converter
from os.path import join as pjoin
from os.path import isdir, isfile, exists
from jupyter_core.paths import jupyter_data_dir
# Mock existence of declarative widgets
DECL_WIDGETS_DIR = pjoin(jupyter_data_dir(), 'nbextensions/urth_widgets/')
DECL_WIDGETS_JS_DIR = pjoin(DECL_WIDGETS_DIR, 'js')
DECL_VIZ_DIR = pjoin(DECL_WIDGETS_DIR, 'components/urth-viz')
DECL_CORE_DIR = pjoin(DECL_WIDGETS_DIR, 'components/urth-core')
BOWER_COMPONENT_DIR = pjoin(jupyter_data_dir(), 'nbextensions/urth_widgets/urth_components/component-a')
class MockHandler(object):
def __init__(self, notebook_dir):
self.notebook_dir = notebook_dir
self.settings = {
'base_url' : '/'
}
self.last_redirect = None
self.tools = None
def redirect(self, location):
self.last_redirect = location
from IPython.core.display import HTML
from glob import glob
# path to local site-packages/jupyterthemes
package_dir = os.path.dirname(os.path.realpath(__file__))
# path to user jupyter-themes dir
user_dir = os.path.join(os.path.expanduser('~'), '.jupyter-themes')
# path to save tempfile with style_less before reading/compiling
tempfile = os.path.join(package_dir, 'tempfile.less')
vimtemp = os.path.join(package_dir, 'vimtemp.less')
# path to install custom.css file (~/.jupyter/custom/)
jupyter_home = jupyter_config_dir()
jupyter_data = jupyter_data_dir()
jupyter_custom = os.path.join(jupyter_home, 'custom')
jupyter_custom_fonts = os.path.join(jupyter_custom, 'fonts')
jupyter_customcss = os.path.join(jupyter_custom, 'custom.css')
jupyter_nbext = os.path.join(jupyter_data, 'nbextensions')
# theme colors, layout, and font directories
layouts_dir = os.path.join(package_dir, 'layout')
styles_dir = os.path.join(package_dir, 'styles')
styles_dir_user = os.path.join(user_dir, 'styles')
fonts_dir = os.path.join(package_dir, 'fonts')
# layout files for notebook, codemirror, cells, mathjax, & vim ext
nb_style = os.path.join(layouts_dir, 'notebook.less')
cm_style = os.path.join(layouts_dir, 'codemirror.less')
# Copyright (c) Jupyter Development Team.
from jupyter_core.paths import jupyter_data_dir
import subprocess
import os
import errno
import stat
PEM_FILE = os.path.join(jupyter_data_dir(), 'notebook.pem')
c = get_config()
c.NotebookApp.ip = '*'
c.NotebookApp.port = 8000
c.NotebookApp.open_browser = False
c.HistoryManager.hist_file = '/tmp/ipython_hist.sqlite'
c.NotebookApp.allow_origin = '*'
#c.NotebookApp.set_default('nbserver_extensions', {})['jupyter_nbextensions_configurator'] = True
c.NotebookApp.trust_xheaders = True
c.NotebookApp.password_required = False
c.NotebookApp.disable_check_xsrf = False
c.NotebookApp.allow_remote_access = True
c.FileCheckpoints.checkpoint_dir = "/tmp"
c.NotebookApp.terminals_enabled = False
c.FileContentsManager.delete_to_trash = False
c.LatexConfig.latex_command = 'pdflatex'
# Set a certificate if USE_HTTPS is set to any value
if 'USE_HTTPS' in os.environ:
if not os.path.isfile(PEM_FILE):
# Ensure PEM_FILE directory exists
dir_name = os.path.dirname(PEM_FILE)
def _cache_default(self):
return os.path.join(jupyter_data_dir(), 'nbgrader_cache')
def _user_kernel_dir_default():
return pjoin(jupyter_data_dir(), 'kernels')
def persistence_root_default(self):
return os.getenv(self.persistence_root_env, jupyter_data_dir())
from jupyter_core.paths import jupyter_data_dir
import os
import sys
c = get_config()
# specify iPython server IP range & listening port
c.NotebookApp.ip = '*'
c.NotebookApp.port = 8133
# do not open browser as Amazon Linux EC2 server instances do not have browsers
c.NotebookApp.open_browser = False
# enable iPython Notebook Extensions
sys.path.append(os.path.join(jupyter_data_dir(), 'extensions'))
c.NotebookApp.extra_template_paths = [
os.path.join(jupyter_data_dir(), 'templates')
]
#--- nbextensions configuration ---
from jupyter_core.paths import jupyter_config_dir, jupyter_data_dir
import os
import sys
sys.path.append(os.path.join(jupyter_data_dir(), 'extensions'))
c = get_config()
c.Exporter.template_path = ['.', os.path.join(jupyter_data_dir(), 'templates')]
#--- nbextensions configuration ---
from jupyter_core.paths import jupyter_config_dir, jupyter_data_dir
#from IPython.utils.path import get_ipython_dir
import os
import sys
sys.path.append(os.path.join(jupyter_data_dir(), 'extensions'))
c = get_config()
#c.NotebookApp.extra_template_paths = [os.path.join(jupyter_data_dir(),'templates') ]
#ipythondir = get_ipython_dir()
ma_extensions = os.path.join(jupyter_data_dir(), 'nbextensions/ma')
sys.path.append(ma_extensions)
c.NotebookApp.server_extensions = ['server_extensions']
c.NotebookApp.extra_template_paths = [
os.path.join(ma_extensions, 'client/templates')
]
#pip3 install pyclustering # ## pyclustering # ### birch # https://pyclustering.github.io/ 여기에서 패키지 파일 다운 받음 # In[24]: import sys sys.executable # In[25]: from jupyter_core.paths import jupyter_data_dir print(jupyter_data_dir()) # In[ ]: # pc1, pc2만 # In[26]: from pyclustering.cluster.birch import birch from pyclustering.cluster import cluster_visualizer from pyclustering.utils import read_sample from pyclustering.samples.definitions import FAMOUS_SAMPLES # list 형태로! X = finalDataFrame.iloc[:, [0, 1]].values.tolist() X
def test_default_cache_dir(self):
dir = default_cache
assert Path(dir) == Path(jupyter_data_dir()) / 'nbgrader_cache'
def test_data_dir():
assert_output('--data-dir', jupyter_data_dir())
from jupyter_core.paths import jupyter_data_dir
import subprocess
import os
import errno
import stat
c = get_config()
c.NotebookApp.ip = '*'
c.NotebookApp.port = 8888
c.NotebookApp.open_browser = False
c.NotebookApp.iopub_data_rate_limit = 10000000000
# Generate a self-signed certificate
if 'GEN_CERT' in os.environ:
dir_name = jupyter_data_dir()
pem_file = os.path.join(dir_name, 'notebook.pem')
try:
os.makedirs(dir_name)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(dir_name):
pass
else:
raise
# Generate a certificate if one doesn't exist on disk
subprocess.check_call([
'openssl', 'req', '-new', '-newkey', 'rsa:2048', '-days', '365',
'-nodes', '-x509', '-subj',
'/C=XX/ST=XX/L=XX/O=generated/CN=generated', '-keyout', pem_file,
'-out', pem_file
])
def _data_dir_default(self):
return jupyter_data_dir()
def install_kernel(kernel_name, ssh_host, **kwargs):
global logger
ssh_key = kwargs.get('ssh_key', None)
jump_server = kwargs.get('jump_server', None)
pre_command = kwargs.get('pre_command', None)
kernel_cmd = kwargs.get('kernel', 'python -m ipykernel')
dry_run = kwargs.get('dry_run', False)
skip_kernel_test = kwargs.get('skip_kernel_test', False)
no_remote_files = kwargs.get('no_remote_files', False)
try:
with ParamikoClient().connect_override(ssh_host, ssh_key, jump_server) as ssh_client:
logger.info('Connection to remote server successfull!')
chan = ssh_client.get_transport().open_session()
chan.get_pty()
cmds = []
if pre_command is not None:
cmds.append(pre_command)
if not skip_kernel_test:
cmds.append('%s --help-all' % kernel_cmd)
cmd = ' && '.join(cmds)
logger.info('Running cmd %s', cmd)
chan.exec_command(cmd)
result = chan.recv_exit_status()
output = ''
data = chan.recv(4096)
while data:
output += data.decode('utf-8')
logger.debug("REMOTE >>> " + data.decode('utf-8'))
data = chan.recv(4096)
if result != 0:
logger.error('CMD %s returned a non-zero exit status on remote server.\n\n%s', cmd, output)
return 1
elif not skip_kernel_test:
for cmd in CMD_ARGS.keys():
if '\n' + cmd not in output:
logger.error('Help message does not specify required argument %s', cmd)
return 1
if dry_run:
logger.info('Test passed, returning without writing kernel specs.')
return 0
if kwargs.get('synchronize', False):
try:
synchronizer = ParamikoSync(ssh_client, **{k: v for k, v in kwargs.items() if k in
('local_folder=', 'remote_folder', 'recursive', 'bi_directional')})
with synchronizer.connect(skip_check=True):
synchronizer.check_remote_sync_folder()
except Exception:
logger.error('Error setting up synchronization on the remote server!', exc_info=True)
return 1
logger.info('Command successful, writing kernel_spec file')
name_spec = dict(
user=ssh_client.username,
host=ssh_client.host,
port=ssh_client.port
)
kernel_name = kernel_name % name_spec
safe_name = ''.join(c if c.isalnum() or c in ('.', '-', '_') else '-' for c in kernel_name).rstrip()
kernel_dir = os.path.join(jupyter_data_dir(), 'kernels', safe_name)
if os.path.isdir(kernel_dir):
logger.error('Kernel directory %s already exists. Choose another name or delete the directory', kernel_dir)
return 1
# Build-up command args to start a kernel
kernel_args = [
sys.executable,
'-m', 'remote_kernel',
'-t', ssh_host
]
if jump_server is not None:
for j in jump_server:
kernel_args += ['-J', j]
if ssh_key is not None:
kernel_args += ['-i', ssh_key]
if pre_command is not None:
kernel_args += ['-pc', pre_command]
if kernel_cmd != 'python -m ipykernel':
kernel_args += ['-k', kernel_cmd]
if no_remote_files:
kernel_args += ['--no-remote-files']
kernel_args += ['-f', '{connection_file}']
# Synchronization config
if kwargs.get('synchronize', False):
kernel_args += ['-s']
if not kwargs.get('recursive', True):
kernel_args += ['--no-recursive']
if kwargs.get('bi_directional', False):
kernel_args += ['--bi-directional']
if kwargs.get('local_folder', 'remote_kernel_sync') != 'remote_kernel_sync':
kernel_args += ['--local-folder', kwargs['local_folder']]
if kwargs.get('remote_folder', 'remote_kernel_sync') != 'remote_kernel_sync':
kernel_args += ['--remote-folder', kwargs['remote_folder']]
kernel_spec = dict(
argv=kernel_args,
language='python',
display_name=kernel_name
)
os.makedirs(kernel_dir)
with open(os.path.join(kernel_dir, 'kernel.json'), mode='w') as kernel_fs:
json.dump(kernel_spec, kernel_fs, indent=2)
resource_dir = get_resource_dir()
for fname in ('logo-32x32.png', 'logo-64x64.png'):
shutil.copy(os.path.join(resource_dir, fname), os.path.join(kernel_dir, fname))
logger.info('Kernel specification installed in %s', kernel_dir)
return 0
except Exception:
logger.error('Error installing kernel specs!', exc_info=True)
return 2
def kernel_dir():
return pjoin(paths.jupyter_data_dir(), 'kernels')
