class _Selection(LabeledWidget, ValueWidget, CoreWidget):
"""Base class for Selection widgets
``options`` can be specified as a list of values, list of (label, value)
tuples, or a dict of {label: value}. The labels are the strings that will be
displayed in the UI, representing the actual Python choices, and should be
unique. If labels are not specified, they are generated from the values.
When programmatically setting the value, a reverse lookup is performed
among the options to check that the value is valid. The reverse lookup uses
the equality operator by default, but another predicate may be provided via
the ``equals`` keyword argument. For example, when dealing with numpy arrays,
one may set equals=np.array_equal.
Only labels are synced (values are converted to/from labels), so the labels should
be unique.
"""
value = Any(help="Selected value").tag(sync=True,
to_json=_value_to_label,
from_json=_label_to_value)
options = Union([List(), Dict()],
help="""List of values, or (label, value) tuples, or a dict of {label: value} pairs that the user can select.
The labels are the strings that will be displayed in the UI, representing the
actual Python choices, and should be unique. If labels are not specified, they
are generated from the values.
The keys are also available as _options_labels.
""")
_options_dict = Dict(read_only=True)
_options_labels = Tuple(read_only=True).tag(sync=True)
_options_values = Tuple(read_only=True)
_model_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_module = Unicode('jupyter-js-widgets').tag(sync=True)
disabled = Bool(help="Enable or disable user changes").tag(sync=True)
def __init__(self, *args, **kwargs):
self.equals = kwargs.pop('equals', lambda x, y: x == y)
super(_Selection, self).__init__(*args, **kwargs)
def _make_options(self, x):
"""Standardize the options list format.
The returned list should be in the format [('label', value), ('label', value), ...].
The input can be
* a Mapping of labels to values
* an iterable of values (of which at least one is not a list or tuple of length 2)
* an iterable with entries that are lists or tuples of the form ('label', value)
"""
# Return a list of key-value pairs where the keys are strings
# If x is a dict, convert it to list format.
if isinstance(x, Mapping):
return [(unicode_type(k), v) for k, v in x.items()]
# If any entry of x is not a list or tuple of length 2, convert
# the entries to unicode for the labels.
for y in x:
if not (isinstance(y, (list, tuple)) and len(y) == 2):
return [(unicode_type(i), i) for i in x]
# x is already in the correct format: a list of 2-tuples.
# The first element of each tuple should be unicode, this might
# not yet be the case.
return [(unicode_type(k), v) for k, v in x]
@validate('options')
def _validate_options(self, proposal):
"""Handles when the options tuple has been changed.
Setting options with a dict implies setting option labels from the keys of the dict.
"""
new = proposal['value']
options = self._make_options(new)
self.set_trait('_options_dict', dict(options))
self.set_trait('_options_labels', [ i[0] for i in options ])
self.set_trait('_options_values', [ i[1] for i in options ])
return new
@observe('options')
def _value_in_options(self, change):
"Ensure the value is an option; if not, set to the first value"
# ensure that the chosen value is still one of the options
if len(self.options) == 0:
self.value = None
else:
try:
_value_to_label(self.value, self)
except KeyError:
self.value = self._options_values[0]
@validate('value')
def _validate_value(self, proposal):
value = proposal['value']
if len(self.options) == 0:
if value is None:
return value
else:
raise TraitError('Invalid selection: empty options list')
else:
try:
_value_to_label(value, self)
return value
except KeyError:
raise TraitError('Invalid selection')
class Button(DOMWidget, CoreWidget):
"""Button widget.
This widget has an `on_click` method that allows you to listen for the
user clicking on the button. The click event itself is stateless.
Parameters
----------
description: str
description displayed next to the button
tooltip: str
tooltip caption of the toggle button
icon: str
font-awesome icon name
"""
_model_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_name = Unicode('ButtonView').tag(sync=True)
_model_name = Unicode('ButtonModel').tag(sync=True)
description = Unicode('Button', help="Button label.").tag(sync=True)
tooltip = Unicode(help="Tooltip caption of the button.").tag(sync=True)
disabled = Bool(False,
help="Enable or disable user changes.").tag(sync=True)
icon = Unicode(
'', help="Font-awesome icon name, without the 'fa-' prefix.").tag(
sync=True)
button_style = CaselessStrEnum(
values=['primary', 'success', 'info', 'warning', 'danger', ''],
default_value='',
help="""Use a predefined styling for the button.""").tag(sync=True)
style = Instance(ButtonStyle).tag(sync=True, **widget_serialization)
@default('style')
def _default_style(self):
return ButtonStyle()
def __init__(self, **kwargs):
super(Button, self).__init__(**kwargs)
self._click_handlers = CallbackDispatcher()
self.on_msg(self._handle_button_msg)
@validate('icon')
def _validate_icon(self, proposal):
"""Strip 'fa-' if necessary'"""
value = proposal['value']
if value.startswith('fa-'):
warnings.warn(
"icons names no longer start with 'fa-', "
"just use the class name itself (for example, 'check' instead of 'fa-check')",
DeprecationWarning)
value = value[3:]
return value
def on_click(self, callback, remove=False):
"""Register a callback to execute when the button is clicked.
The callback will be called with one argument, the clicked button
widget instance.
Parameters
----------
remove: bool (optional)
Set to true to remove the callback from the list of callbacks.
"""
self._click_handlers.register_callback(callback, remove=remove)
def _handle_button_msg(self, _, content, buffers):
"""Handle a msg from the front-end.
Parameters
----------
content: dict
Content of the msg.
"""
if content.get('event', '') == 'click':
self._click_handlers(self)
class OAuthenticator(Authenticator):
"""Base class for OAuthenticators
Subclasses must override:
login_service (string identifying the service provider)
login_handler (likely a subclass of OAuthLoginHandler)
authenticate (method takes one arg - the request handler handling the oauth callback)
"""
scope = List(Unicode(), config=True,
help="""The OAuth scopes to request.
See the OAuth documentation of your OAuth provider for options.
For GitHub in particular, you can see github_scopes.md in this repo.
"""
)
login_service = 'override in subclass'
oauth_callback_url = Unicode(
os.getenv('OAUTH_CALLBACK_URL', ''),
config=True,
help="""Callback URL to use.
Typically `https://{host}/hub/oauth_callback`"""
)
client_id_env = ''
client_id = Unicode(config=True)
def _client_id_default(self):
if self.client_id_env:
client_id = os.getenv(self.client_id_env, '')
if client_id:
return client_id
return os.getenv('OAUTH_CLIENT_ID', '')
client_secret_env = ''
client_secret = Unicode(config=True)
def _client_secret_default(self):
if self.client_secret_env:
client_secret = os.getenv(self.client_secret_env, '')
if client_secret:
return client_secret
return os.getenv('OAUTH_CLIENT_SECRET', '')
validate_server_cert_env = 'OAUTH_TLS_VERIFY'
validate_server_cert = Bool(config=True)
def _validate_server_cert_default(self):
env_value = os.getenv(self.validate_server_cert_env, '')
if env_value == '0':
return False
else:
return True
def login_url(self, base_url):
return url_path_join(base_url, 'oauth_login')
login_handler = "Specify login handler class in subclass"
callback_handler = OAuthCallbackHandler
def get_callback_url(self, handler=None):
"""Get my OAuth redirect URL
Either from config or guess based on the current request.
"""
if self.oauth_callback_url:
return self.oauth_callback_url
elif handler:
return guess_callback_uri(
handler.request.protocol,
handler.request.host,
handler.hub.server.base_url
)
else:
raise ValueError("Specify callback oauth_callback_url or give me a handler to guess with")
def get_handlers(self, app):
return [
(r'/oauth_login', self.login_handler),
(r'/oauth_callback', self.callback_handler),
]
async def authenticate(self, handler, data=None):
raise NotImplementedError()
class LDAPAuthenticator(Authenticator):
server_address = Unicode(config=True,
help='Address of LDAP server to contact')
server_port = Int(
config=True,
help='Port on which to contact LDAP server',
)
def _server_port_default(self):
if self.use_ssl:
return 636 # default SSL port for LDAP
else:
return 389 # default plaintext port for LDAP
use_ssl = Bool(True,
config=True,
help='Use SSL to encrypt connection to LDAP server')
bind_dn_template = Unicode(config=True,
help="""
Template from which to construct the full dn
when authenticating to LDAP. {username} is replaced
with the actual username.
Example:
uid={username},ou=people,dc=wikimedia,dc=org
""")
allowed_groups = List(
config=True,
help="List of LDAP Group DNs whose members are allowed access")
valid_username_regex = Unicode(
r'^[a-z][.a-z0-9_-]*$',
config=True,
help="""Regex to use to validate usernames before sending to LDAP
Also acts as a security measure to prevent LDAP injection. If you
are customizing this, be careful to ensure that attempts to do LDAP
injection are rejected by your customization
""")
lookup_dn = Bool(False,
config=True,
help='Look up the user\'s DN based on an attribute')
user_search_base = Unicode(
config=True,
help="""Base for looking up user accounts in the directory.
Example:
ou=people,dc=wikimedia,dc=org
""")
user_attribute = Unicode(
config=True,
help="""LDAP attribute that stores the user's username.
For most LDAP servers, this is uid. For Active Directory, it is
sAMAccountName.
""")
@gen.coroutine
def authenticate(self, handler, data):
username = data['username']
password = data['password']
# Protect against invalid usernames as well as LDAP injection attacks
if not re.match(self.valid_username_regex, username):
self.log.warn('Invalid username')
return None
# No empty passwords!
if password is None or password.strip() == '':
self.log.warn('Empty password')
return None
userdn = self.bind_dn_template.format(username=username)
server = ldap3.Server(self.server_address,
port=self.server_port,
use_ssl=self.use_ssl)
conn = ldap3.Connection(server, user=userdn, password=password)
if conn.bind():
if self.allowed_groups:
if self.lookup_dn:
# In some cases, like AD, we don't bind with the DN, and need to discover it.
conn.search(search_base=self.user_search_base,
search_scope=ldap3.SUBTREE,
search_filter='({userattr}={username})'.format(
userattr=self.user_attribute,
username=username),
attributes=[self.user_attribute])
if len(conn.response) == 0:
self.log.warn(
'User with {userattr}={username} not found in directory'
.format(userattr=self.user_attribute,
username=username))
return None
userdn = conn.response[0]['dn']
for group in self.allowed_groups:
groupfilter = ('(|'
'(member={userdn})'
'(uniqueMember={userdn})'
'(memberUid={uid})'
')').format(userdn=userdn, uid=username)
groupattributes = ['member', 'uniqueMember', 'memberUid']
if conn.search(group,
search_scope=ldap3.BASE,
search_filter=groupfilter,
attributes=groupattributes):
return username
# If we reach here, then none of the groups matched
self.log.warn(
'User {username} not in any of the allowed groups'.format(
username=userdn))
return None
else:
return username
else:
self.log.warn('Invalid password for user {username}'.format(
username=userdn, ))
return None
class Voila(Application):
name = 'voila'
version = __version__
examples = 'voila example.ipynb --port 8888'
flags = {
'debug': ({'Voila': {'log_level': logging.DEBUG}}, _("Set the log level to logging.DEBUG")),
'no-browser': ({'Voila': {'open_browser': False}}, _('Don\'t open the notebook in a browser after startup.'))
}
description = Unicode(
"""voila [OPTIONS] NOTEBOOK_FILENAME
This launches a stand-alone server for read-only notebooks.
"""
)
option_description = Unicode(
"""
notebook_path:
File name of the Jupyter notebook to display.
"""
)
notebook_filename = Unicode()
port = Integer(
8866,
config=True,
help=_(
'Port of the voila server. Default 8866.'
)
)
autoreload = Bool(
False,
config=True,
help=_(
'Will autoreload to server and the page when a template, js file or Python code changes'
)
)
root_dir = Unicode(config=True, help=_('The directory to use for notebooks.'))
static_root = Unicode(
STATIC_ROOT,
config=True,
help=_(
'Directory holding static assets (HTML, JS and CSS files).'
)
)
aliases = {
'port': 'Voila.port',
'static': 'Voila.static_root',
'strip_sources': 'VoilaConfiguration.strip_sources',
'autoreload': 'Voila.autoreload',
'template': 'VoilaConfiguration.template',
'theme': 'VoilaConfiguration.theme',
'base_url': 'Voila.base_url',
'server_url': 'Voila.server_url',
'enable_nbextensions': 'VoilaConfiguration.enable_nbextensions'
}
classes = [
VoilaConfiguration,
VoilaExecutor,
VoilaExporter,
VoilaCSSPreprocessor
]
connection_dir_root = Unicode(
config=True,
help=_(
'Location of temporry connection files. Defaults '
'to system `tempfile.gettempdir()` value.'
)
)
connection_dir = Unicode()
base_url = Unicode(
'/',
config=True,
help=_(
'Path for voila API calls. If server_url is unset, this will be \
used for both the base route of the server and the client. \
If server_url is set, the server will server the routes prefixed \
by server_url, while the client will prefix by base_url (this is \
useful in reverse proxies).'
)
)
server_url = Unicode(
None,
config=True,
allow_none=True,
help=_(
'Path to prefix to voila API handlers. Leave unset to default to base_url'
)
)
notebook_path = Unicode(
None,
config=True,
allow_none=True,
help=_(
'path to notebook to serve with voila'
)
)
nbconvert_template_paths = List(
[],
config=True,
help=_(
'path to nbconvert templates'
)
)
template_paths = List(
[],
allow_none=True,
config=True,
help=_(
'path to nbconvert templates'
)
)
static_paths = List(
[STATIC_ROOT],
config=True,
help=_(
'paths to static assets'
)
)
port_retries = Integer(50, config=True,
help=_("The number of additional ports to try if the specified port is not available.")
)
ip = Unicode('localhost', config=True,
help=_("The IP address the notebook server will listen on."))
open_browser = Bool(True, config=True,
help=_("""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
"""))
browser = Unicode(u'', config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
webbrowser_open_new = Integer(2, config=True,
help=_("""Specify Where to open the notebook on startup. This is the
`new` argument passed to the standard library method `webbrowser.open`.
The behaviour is not guaranteed, but depends on browser support. Valid
values are:
- 2 opens a new tab,
- 1 opens a new window,
- 0 opens in an existing window.
See the `webbrowser.open` documentation for details.
"""))
custom_display_url = Unicode(u'', config=True,
help=_("""Override URL shown to users.
Replace actual URL, including protocol, address, port and base URL,
with the given value when displaying URL to the users. Do not change
the actual connection URL. If authentication token is enabled, the
token is added to the custom URL automatically.
This option is intended to be used when the URL to display to the user
cannot be determined reliably by the Jupyter notebook server (proxified
or containerized setups for example)."""))
@property
def display_url(self):
if self.custom_display_url:
url = self.custom_display_url
if not url.endswith('/'):
url += '/'
else:
if self.ip in ('', '0.0.0.0'):
ip = "%s" % socket.gethostname()
else:
ip = self.ip
url = self._url(ip)
# TODO: do we want to have the token?
# if self.token:
# # Don't log full token if it came from config
# token = self.token if self._token_generated else '...'
# url = (url_concat(url, {'token': token})
# + '\n or '
# + url_concat(self._url('127.0.0.1'), {'token': token}))
return url
@property
def connection_url(self):
ip = self.ip if self.ip else 'localhost'
return self._url(ip)
def _url(self, ip):
# TODO: https / certfile
# proto = 'https' if self.certfile else 'http'
proto = 'http'
return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
config_file_paths = List(
Unicode(),
config=True,
help=_(
'Paths to search for voila.(py|json)'
)
)
tornado_settings = Dict(
{},
config=True,
help=_(
'Extra settings to apply to tornado application, e.g. headers, ssl, etc'
)
)
@default('config_file_paths')
def _config_file_paths_default(self):
return [os.getcwd()] + jupyter_config_path()
@default('connection_dir_root')
def _default_connection_dir(self):
connection_dir = tempfile.gettempdir()
self.log.info('Using %s to store connection files' % connection_dir)
return connection_dir
@default('log_level')
def _default_log_level(self):
return logging.INFO
# similar to NotebookApp, except no extra path
@property
def nbextensions_path(self):
"""The path to look for Javascript notebook extensions"""
path = jupyter_path('nbextensions')
# FIXME: remove IPython nbextensions path after a migration period
try:
from IPython.paths import get_ipython_dir
except ImportError:
pass
else:
path.append(os.path.join(get_ipython_dir(), 'nbextensions'))
return path
@default('root_dir')
def _default_root_dir(self):
if self.notebook_path:
return os.path.dirname(os.path.abspath(self.notebook_path))
else:
return getcwd()
def initialize(self, argv=None):
self.log.debug("Searching path %s for config files", self.config_file_paths)
# to make config_file_paths settable via cmd line, we first need to parse it
super(Voila, self).initialize(argv)
if len(self.extra_args) == 1:
arg = self.extra_args[0]
# I am not sure why we need to check if self.notebook_path is set, can we get rid of this?
if not self.notebook_path:
if os.path.isdir(arg):
self.root_dir = arg
elif os.path.isfile(arg):
self.notebook_path = arg
else:
raise ValueError('argument is neither a file nor a directory: %r' % arg)
elif len(self.extra_args) != 0:
raise ValueError('provided more than 1 argument: %r' % self.extra_args)
# then we load the config
self.load_config_file('voila', path=self.config_file_paths)
# common configuration options between the server extension and the application
self.voila_configuration = VoilaConfiguration(parent=self)
self.setup_template_dirs()
signal.signal(signal.SIGTERM, self._handle_signal_stop)
def setup_template_dirs(self):
if self.voila_configuration.template:
collect_template_paths(
self.nbconvert_template_paths,
self.static_paths,
self.template_paths,
self.voila_configuration.template)
# look for possible template-related config files
template_conf_dir = [os.path.join(k, '..') for k in self.nbconvert_template_paths]
conf_paths = [os.path.join(d, 'conf.json') for d in template_conf_dir]
for p in conf_paths:
# see if config file exists
if os.path.exists(p):
# load the template-related config
with open(p) as json_file:
conf = json.load(json_file)
# update the overall config with it, preserving CLI config priority
if 'traitlet_configuration' in conf:
recursive_update(conf['traitlet_configuration'], self.voila_configuration.config.VoilaConfiguration)
# pass merged config to overall voila config
self.voila_configuration.config.VoilaConfiguration = Config(conf['traitlet_configuration'])
self.log.debug('using template: %s', self.voila_configuration.template)
self.log.debug('nbconvert template paths:\n\t%s', '\n\t'.join(self.nbconvert_template_paths))
self.log.debug('template paths:\n\t%s', '\n\t'.join(self.template_paths))
self.log.debug('static paths:\n\t%s', '\n\t'.join(self.static_paths))
if self.notebook_path and not os.path.exists(self.notebook_path):
raise ValueError('Notebook not found: %s' % self.notebook_path)
def _handle_signal_stop(self, sig, frame):
self.log.info('Handle signal %s.' % sig)
self.ioloop.add_callback_from_signal(self.ioloop.stop)
def start(self):
self.connection_dir = tempfile.mkdtemp(
prefix='voila_',
dir=self.connection_dir_root
)
self.log.info('Storing connection files in %s.' % self.connection_dir)
self.log.info('Serving static files from %s.' % self.static_root)
self.kernel_spec_manager = KernelSpecManager(
parent=self
)
self.kernel_manager = AsyncMappingKernelManager(
parent=self,
connection_dir=self.connection_dir,
kernel_spec_manager=self.kernel_spec_manager,
allowed_message_types=[
'comm_open',
'comm_close',
'comm_msg',
'comm_info_request',
'kernel_info_request',
'shutdown_request'
]
)
jenv_opt = {"autoescape": True} # we might want extra options via cmd line like notebook server
env = jinja2.Environment(loader=jinja2.FileSystemLoader(self.template_paths), extensions=['jinja2.ext.i18n'], **jenv_opt)
nbui = gettext.translation('nbui', localedir=os.path.join(ROOT, 'i18n'), fallback=True)
env.install_gettext_translations(nbui, newstyle=False)
self.contents_manager = LargeFileManager(parent=self)
# we create a config manager that load both the serverconfig and nbconfig (classical notebook)
read_config_path = [os.path.join(p, 'serverconfig') for p in jupyter_config_path()]
read_config_path += [os.path.join(p, 'nbconfig') for p in jupyter_config_path()]
self.config_manager = ConfigManager(parent=self, read_config_path=read_config_path)
# default server_url to base_url
self.server_url = self.server_url or self.base_url
self.app = tornado.web.Application(
base_url=self.base_url,
server_url=self.server_url or self.base_url,
kernel_manager=self.kernel_manager,
kernel_spec_manager=self.kernel_spec_manager,
allow_remote_access=True,
autoreload=self.autoreload,
voila_jinja2_env=env,
jinja2_env=env,
static_path='/',
server_root_dir='/',
contents_manager=self.contents_manager,
config_manager=self.config_manager
)
self.app.settings.update(self.tornado_settings)
handlers = []
handlers.extend([
(url_path_join(self.server_url, r'/api/kernels/%s' % _kernel_id_regex), KernelHandler),
(url_path_join(self.server_url, r'/api/kernels/%s/channels' % _kernel_id_regex), ZMQChannelsHandler),
(
url_path_join(self.server_url, r'/voila/static/(.*)'),
MultiStaticFileHandler,
{
'paths': self.static_paths,
'default_filename': 'index.html'
}
)
])
# Serving notebook extensions
if self.voila_configuration.enable_nbextensions:
handlers.append(
(
url_path_join(self.server_url, r'/voila/nbextensions/(.*)'),
FileFindHandler,
{
'path': self.nbextensions_path,
'no_cache_paths': ['/'], # don't cache anything in nbextensions
},
)
)
handlers.append(
(
url_path_join(self.server_url, r'/voila/files/(.*)'),
WhiteListFileHandler,
{
'whitelist': self.voila_configuration.file_whitelist,
'blacklist': self.voila_configuration.file_blacklist,
'path': self.root_dir,
},
)
)
tree_handler_conf = {
'voila_configuration': self.voila_configuration
}
if self.notebook_path:
handlers.append((
url_path_join(self.server_url, r'/(.*)'),
VoilaHandler,
{
'notebook_path': os.path.relpath(self.notebook_path, self.root_dir),
'nbconvert_template_paths': self.nbconvert_template_paths,
'config': self.config,
'voila_configuration': self.voila_configuration
}
))
else:
self.log.debug('serving directory: %r', self.root_dir)
handlers.extend([
(self.server_url, VoilaTreeHandler, tree_handler_conf),
(url_path_join(self.server_url, r'/voila/tree' + path_regex),
VoilaTreeHandler, tree_handler_conf),
(url_path_join(self.server_url, r'/voila/render/(.*)'),
VoilaHandler,
{
'nbconvert_template_paths': self.nbconvert_template_paths,
'config': self.config,
'voila_configuration': self.voila_configuration
}),
])
self.app.add_handlers('.*$', handlers)
self.listen()
def stop(self):
shutil.rmtree(self.connection_dir)
run_sync(self.kernel_manager.shutdown_all())
def random_ports(self, port, n):
"""Generate a list of n random ports near the given port.
The first 5 ports will be sequential, and the remaining n-5 will be
randomly selected in the range [port-2*n, port+2*n].
"""
for i in range(min(5, n)):
yield port + i
for i in range(n-5):
yield max(1, port + random.randint(-2*n, 2*n))
def listen(self):
for port in self.random_ports(self.port, self.port_retries+1):
try:
self.app.listen(port)
self.port = port
self.log.info('Voila is running at:\n%s' % self.display_url)
except socket.error as e:
if e.errno == errno.EADDRINUSE:
self.log.info(_('The port %i is already in use, trying another port.') % port)
continue
elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)):
self.log.warning(_("Permission to listen on port %i denied") % port)
continue
else:
raise
else:
self.port = port
success = True
break
if not success:
self.log.critical(_('ERROR: the voila server could not be started because '
'no available port could be found.'))
self.exit(1)
if self.open_browser:
self.launch_browser()
self.ioloop = tornado.ioloop.IOLoop.current()
try:
self.ioloop.start()
except KeyboardInterrupt:
self.log.info('Stopping...')
finally:
self.stop()
def launch_browser(self):
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warning(_('No web browser found: %s.') % e)
browser = None
if not browser:
return
uri = self.base_url
fd, open_file = tempfile.mkstemp(suffix='.html')
# Write a temporary file to open in the browser
with io.open(fd, 'w', encoding='utf-8') as fh:
# TODO: do we want to have the token?
# if self.token:
# url = url_concat(url, {'token': self.token})
url = url_path_join(self.connection_url, uri)
jinja2_env = self.app.settings['jinja2_env']
template = jinja2_env.get_template('browser-open.html')
fh.write(template.render(open_url=url, base_url=url))
def target():
return browser.open(urljoin('file:', pathname2url(open_file)), new=self.webbrowser_open_new)
threading.Thread(target=target).start()
class Authenticator(LoggingConfigurable):
"""Base class for implementing an authentication provider for JupyterHub"""
db = Any()
enable_auth_state = Bool(
False,
config=True,
help="""Enable persisting auth_state (if available).
auth_state will be encrypted and stored in the Hub's database.
This can include things like authentication tokens, etc.
to be passed to Spawners as environment variables.
Encrypting auth_state requires the cryptography package.
Additionally, the JUPYTERHUB_CRYPTO_KEY envirionment variable must
contain one (or more, separated by ;) 32B encryption keys.
These can be either base64 or hex-encoded.
If encryption is unavailable, auth_state cannot be persisted.
New in JupyterHub 0.8
""",
)
admin_users = Set(help="""
Set of users that will have admin rights on this JupyterHub.
Admin users have extra privileges:
- Use the admin panel to see list of users logged in
- Add / remove users in some authenticators
- Restart / halt the hub
- Start / stop users' single-user servers
- Can access each individual users' single-user server (if configured)
Admin access should be treated the same way root access is.
Defaults to an empty set, in which case no user has admin access.
""").tag(config=True)
whitelist = Set(help="""
Whitelist of usernames that are allowed to log in.
Use this with supported authenticators to restrict which users can log in. This is an
additional whitelist that further restricts users, beyond whatever restrictions the
authenticator has in place.
If empty, does not perform any additional restriction.
""").tag(config=True)
@observe('whitelist')
def _check_whitelist(self, change):
short_names = [name for name in change['new'] if len(name) <= 1]
if short_names:
sorted_names = sorted(short_names)
single = ''.join(sorted_names)
string_set_typo = "set('%s')" % single
self.log.warning(
"whitelist contains single-character names: %s; did you mean set([%r]) instead of %s?",
sorted_names[:8],
single,
string_set_typo,
)
custom_html = Unicode(help="""
HTML form to be overridden by authenticators if they want a custom authentication form.
Defaults to an empty string, which shows the default username/password form.
""")
login_service = Unicode(help="""
Name of the login service that this authenticator is providing using to authenticate users.
Example: GitHub, MediaWiki, Google, etc.
Setting this value replaces the login form with a "Login with <login_service>" button.
Any authenticator that redirects to an external service (e.g. using OAuth) should set this.
""")
username_pattern = Unicode(help="""
Regular expression pattern that all valid usernames must match.
If a username does not match the pattern specified here, authentication will not be attempted.
If not set, allow any username.
""").tag(config=True)
@observe('username_pattern')
def _username_pattern_changed(self, change):
if not change['new']:
self.username_regex = None
self.username_regex = re.compile(change['new'])
username_regex = Any(help="""
Compiled regex kept in sync with `username_pattern`
""")
def validate_username(self, username):
"""Validate a normalized username
Return True if username is valid, False otherwise.
"""
if not self.username_regex:
return True
return bool(self.username_regex.match(username))
username_map = Dict(
help="""Dictionary mapping authenticator usernames to JupyterHub users.
Primarily used to normalize OAuth user names to local users.
""").tag(config=True)
delete_invalid_users = Bool(
False,
help="""Delete any users from the database that do not pass validation
When JupyterHub starts, `.add_user` will be called
on each user in the database to verify that all users are still valid.
If `delete_invalid_users` is True,
any users that do not pass validation will be deleted from the database.
Use this if users might be deleted from an external system,
such as local user accounts.
If False (default), invalid users remain in the Hub's database
and a warning will be issued.
This is the default to avoid data loss due to config changes.
""")
def normalize_username(self, username):
"""Normalize the given username and return it
Override in subclasses if usernames need different normalization rules.
The default attempts to lowercase the username and apply `username_map` if it is
set.
"""
username = username.lower()
username = self.username_map.get(username, username)
return username
def check_whitelist(self, username):
"""Check if a username is allowed to authenticate based on whitelist configuration
Return True if username is allowed, False otherwise.
No whitelist means any username is allowed.
Names are normalized *before* being checked against the whitelist.
"""
if not self.whitelist:
# No whitelist means any name is allowed
return True
return username in self.whitelist
@gen.coroutine
def get_authenticated_user(self, handler, data):
"""Authenticate the user who is attempting to log in
Returns user dict if successful, None otherwise.
This calls `authenticate`, which should be overridden in subclasses,
normalizes the username if any normalization should be done,
and then validates the name in the whitelist.
This is the outer API for authenticating a user.
Subclasses should not override this method.
The various stages can be overridden separately:
- `authenticate` turns formdata into a username
- `normalize_username` normalizes the username
- `check_whitelist` checks against the user whitelist
.. versionchanged:: 0.8
return dict instead of username
"""
authenticated = yield self.authenticate(handler, data)
if authenticated is None:
return
if isinstance(authenticated, dict):
if 'name' not in authenticated:
raise ValueError("user missing a name: %r" % authenticated)
else:
authenticated = {
'name': authenticated,
}
authenticated.setdefault('auth_state', None)
# normalize the username
authenticated['name'] = username = self.normalize_username(
authenticated['name'])
if not self.validate_username(username):
self.log.warning("Disallowing invalid username %r.", username)
return
whitelist_pass = yield gen.maybe_future(self.check_whitelist(username))
if whitelist_pass:
return authenticated
else:
self.log.warning("User %r not in whitelist.", username)
return
@gen.coroutine
def authenticate(self, handler, data):
"""Authenticate a user with login form data
This must be a tornado gen.coroutine.
It must return the username on successful authentication,
and return None on failed authentication.
Checking the whitelist is handled separately by the caller.
.. versionchanged:: 0.8
Allow `authenticate` to return a dict containing auth_state.
Args:
handler (tornado.web.RequestHandler): the current request handler
data (dict): The formdata of the login form.
The default form has 'username' and 'password' fields.
Returns:
user (str or dict or None): The username of the authenticated user,
or None if Authentication failed.
If the Authenticator has state associated with the user,
it can return a dict with the keys 'name' and 'auth_state',
where 'name' is the username and 'auth_state' is a dictionary
of auth state that will be persisted.
"""
def pre_spawn_start(self, user, spawner):
"""Hook called before spawning a user's server
Can be used to do auth-related startup, e.g. opening PAM sessions.
"""
def post_spawn_stop(self, user, spawner):
"""Hook called after stopping a user container
Can be used to do auth-related cleanup, e.g. closing PAM sessions.
"""
def add_user(self, user):
"""Hook called when a user is added to JupyterHub
This is called:
- When a user first authenticates
- When the hub restarts, for all users.
This method may be a coroutine.
By default, this just adds the user to the whitelist.
Subclasses may do more extensive things, such as adding actual unix users,
but they should call super to ensure the whitelist is updated.
Note that this should be idempotent, since it is called whenever the hub restarts
for all users.
Args:
user (User): The User wrapper object
"""
if not self.validate_username(user.name):
raise ValueError("Invalid username: %s" % user.name)
if self.whitelist:
self.whitelist.add(user.name)
def delete_user(self, user):
"""Hook called when a user is deleted
Removes the user from the whitelist.
Subclasses should call super to ensure the whitelist is updated.
Args:
user (User): The User wrapper object
"""
self.whitelist.discard(user.name)
auto_login = Bool(False,
config=True,
help="""Automatically begin the login process
rather than starting with a "Login with..." link at `/hub/login`
To work, `.login_url()` must give a URL other than the default `/hub/login`,
such as an oauth handler or another automatic login handler,
registered with `.get_handlers()`.
.. versionadded:: 0.8
""")
def login_url(self, base_url):
"""Override this when registering a custom login handler
Generally used by authenticators that do not use simple form-based authentication.
The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args:
base_url (str): the base URL of the Hub (e.g. /hub/)
Returns:
str: The login URL, e.g. '/hub/login'
"""
return url_path_join(base_url, 'login')
def logout_url(self, base_url):
"""Override when registering a custom logout handler
The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args:
base_url (str): the base URL of the Hub (e.g. /hub/)
Returns:
str: The logout URL, e.g. '/hub/logout'
"""
return url_path_join(base_url, 'logout')
def get_handlers(self, app):
"""Return any custom handlers the authenticator needs to register
Used in conjugation with `login_url` and `logout_url`.
Args:
app (JupyterHub Application):
the application object, in case it needs to be accessed for info.
Returns:
handlers (list):
list of ``('/url', Handler)`` tuples passed to tornado.
The Hub prefix is added to any URLs.
"""
return [
('/login', LoginHandler),
]
class PAMAuthenticator(LocalAuthenticator):
"""Authenticate local UNIX users with PAM"""
encoding = Unicode('utf8',
help="""
The text encoding to use when communicating with PAM
""").tag(config=True)
service = Unicode('login',
help="""
The name of the PAM service to use for authentication
""").tag(config=True)
open_sessions = Bool(True,
help="""
Whether to open a new PAM session when spawners are started.
This may trigger things like mounting shared filsystems,
loading credentials, etc. depending on system configuration,
but it does not always work.
If any errors are encountered when opening/closing PAM sessions,
this is automatically set to False.
""").tag(config=True)
def __init__(self, **kwargs):
if pamela is None:
raise _pamela_error from None
super().__init__(**kwargs)
@gen.coroutine
def authenticate(self, handler, data):
"""Authenticate with PAM, and return the username if login is successful.
Return None otherwise.
"""
username = data['username']
try:
pamela.authenticate(username,
data['password'],
service=self.service)
except pamela.PAMError as e:
if handler is not None:
self.log.warning("PAM Authentication failed (%s@%s): %s",
username, handler.request.remote_ip, e)
else:
self.log.warning("PAM Authentication failed: %s", e)
else:
return username
def pre_spawn_start(self, user, spawner):
"""Open PAM session for user if so configured"""
if not self.open_sessions:
return
try:
pamela.open_session(user.name, service=self.service)
except pamela.PAMError as e:
self.log.warning("Failed to open PAM session for %s: %s",
user.name, e)
self.log.warning("Disabling PAM sessions from now on.")
self.open_sessions = False
def post_spawn_stop(self, user, spawner):
"""Close PAM session for user if we were configured to opened one"""
if not self.open_sessions:
return
try:
pamela.close_session(user.name, service=self.service)
except pamela.PAMError as e:
self.log.warning("Failed to close PAM session for %s: %s",
user.name, e)
self.log.warning("Disabling PAM sessions from now on.")
self.open_sessions = False
class InspectReply(Reply, MimeBundle):
found = Bool()
class GenericOAuthenticator(OAuthenticator):
login_service = Unicode("OAuth 2.0", config=True)
extra_params = Dict(help="Extra parameters for first POST request").tag(config=True)
username_key = Union(
[Unicode(os.environ.get('OAUTH2_USERNAME_KEY', 'username')), Callable()],
config=True,
help="""
Userdata username key from returned json for USERDATA_URL.
Can be a string key name or a callable that accepts the returned
json (as a dict) and returns the username. The callable is useful
e.g. for extracting the username from a nested object in the
response.
""",
)
userdata_params = Dict(
help="Userdata params to get user data login information"
).tag(config=True)
userdata_method = Unicode(
os.environ.get('OAUTH2_USERDATA_METHOD', 'GET'),
config=True,
help="Userdata method to get user data login information",
)
userdata_token_method = Unicode(
os.environ.get('OAUTH2_USERDATA_REQUEST_TYPE', 'header'),
config=True,
help="Method for sending access token in userdata request. Supported methods: header, url. Default: header",
)
tls_verify = Bool(
os.environ.get('OAUTH2_TLS_VERIFY', 'True').lower() in {'true', '1'},
config=True,
help="Disable TLS verification on http request",
)
basic_auth = Bool(
os.environ.get('OAUTH2_BASIC_AUTH', 'True').lower() in {'true', '1'},
config=True,
help="Disable basic authentication for access token request",
)
def http_client(self):
return AsyncHTTPClient(force_instance=True, defaults=dict(validate_cert=self.tls_verify))
async def authenticate(self, handler, data=None):
code = handler.get_argument("code")
# TODO: Configure the curl_httpclient for tornado
http_client = self.http_client()
params = dict(
redirect_uri=self.get_callback_url(handler),
code=code,
grant_type='authorization_code',
)
params.update(self.extra_params)
if self.token_url:
url = self.token_url
else:
raise ValueError("Please set the $OAUTH2_TOKEN_URL environment variable")
headers = {"Accept": "application/json", "User-Agent": "JupyterHub"}
if self.basic_auth:
b64key = base64.b64encode(
bytes("{}:{}".format(self.client_id, self.client_secret), "utf8")
)
headers.update({"Authorization": "Basic {}".format(b64key.decode("utf8"))})
req = HTTPRequest(
url,
method="POST",
headers=headers,
body=urllib.parse.urlencode(params),
)
resp = await http_client.fetch(req)
resp_json = json.loads(resp.body.decode('utf8', 'replace'))
access_token = resp_json['access_token']
refresh_token = resp_json.get('refresh_token', None)
token_type = resp_json['token_type']
scope = resp_json.get('scope', '')
if isinstance(scope, str):
scope = scope.split(' ')
# Determine who the logged in user is
headers = {
"Accept": "application/json",
"User-Agent": "JupyterHub",
"Authorization": "{} {}".format(token_type, access_token),
}
if self.userdata_url:
url = url_concat(self.userdata_url, self.userdata_params)
else:
raise ValueError("Please set the OAUTH2_USERDATA_URL environment variable")
if self.userdata_token_method == "url":
url = url_concat(self.userdata_url, dict(access_token=access_token))
req = HTTPRequest(
url,
method=self.userdata_method,
headers=headers,,allow_nonstandard_methods=True
)
resp = await http_client.fetch(req)
resp_json = json.loads(resp.body.decode('utf8', 'replace'))
if callable(self.username_key):
name = self.username_key(resp_json)
else:
name = resp_json.get(self.username_key)
if not name:
self.log.error(
"OAuth user contains no key %s: %s", self.username_key, resp_json
)
return
return {
'name': name,
'auth_state': {
'access_token': access_token,
'refresh_token': refresh_token,
'oauth_user': resp_json,
'scope': scope,
},
}
class AxisAtController(Manipulator):
_StatusMap = {
'I': 'axis is initialized',
'O': 'axis is disabled',
'R': 'axis initialised and ready',
'T': 'axis is positioning in trapezoidal profile',
'S': 'axis is positioning in S-curve profile',
'V': 'axis is operating in velocity mode',
'P': 'reference motion is in progress',
'F': 'axis is releasing a limit switch',
'J': 'axis is operating in joystick mode',
'L': 'axis has been disabled after approaching a hardware limit switch'
' (MINSTOP, MAXSTOP)',
'B': 'axis has been stopped after approaching a brake switch (MINDEC, '
'MAXDEC)',
'A': 'axis has been disabled after limit switch error',
'M': 'axis has been disabled after motion controller error',
'Z': 'axis has been disabled after timeout error',
'H': 'phase initialization activ (step motor axis)',
'U': 'axis is not released.',
'E': 'axis has been disabled after motion error',
'W': 'axis is positioning in trapezoidal profile with WMS',
'X': 'axis is positioning in S-curve profile with WMS',
'Y': 'axis is operating in velocity mode with WMS',
'C': 'axis is operating in velocity mode with continuous-path control',
'?': 'error, unknown status of axis',
}
_movingStates = ['T', 'S', 'V', 'P', 'W', 'X', 'Y', 'C']
statusMessage = Unicode(read_only=True).tag(name="Status")
limitSwitchActive = Bool(read_only=True).tag(name="Limit switch active")
motorPowerStageError = Bool(read_only=True).tag(
name="Motor power stage error")
def __init__(self,
connection=None,
axis=1,
pitch=Q_(1),
objectName=None,
loop=None):
""" Axis `axis` at connection `connection`. Has pitch `pitch` in units
of 'full step count/length'."""
super().__init__(objectName=objectName, loop=loop)
self.connection = connection
self.axis = axis
self._status = '?'
self.set_trait('statusMessage', self._StatusMap[self._status])
self._isMovingFuture = asyncio.Future()
self._isMovingFuture.set_result(None)
self.prefPosUnit = (ureg.count / pitch).units
self.prefVelocUnit = (ureg.count / ureg.s / pitch).units
self.setPreferredUnits(self.prefPosUnit, self.prefVelocUnit)
self._microstepResolution = 50
self._pitch = pitch
self.define_context()
if not pitch.dimensionless:
self.velocity = Q_(1, 'mm/s')
else:
self.velocity = Q_(2000, 'count/s')
def define_context(self):
self.contextName = 'Owis-{}'.format(uuid.uuid4())
context = Context(self.contextName)
if not self._pitch.dimensionless:
context.add_transformation(
'', '[length]', lambda ureg, x: x /
(self._pitch * self._microstepResolution))
context.add_transformation(
'[length]', '[]',
lambda ureg, x: x * self._pitch * self._microstepResolution)
context.add_transformation(
'1/[time]', '[length]/[time]', lambda ureg, x: x /
(self._pitch * self._microstepResolution))
context.add_transformation(
'[length]/[time]', '1/[time]',
lambda ureg, x: x * self._pitch * self._microstepResolution)
ureg.add_context(context)
async def __aenter__(self):
await super().__aenter__()
await self.send("absol" + str(self.axis))
self._microstepResolution = await self.queryAxisVariable('mcstp')
self._updateFuture = ensure_weakly_binding_future(self.updateStatus)
return self
async def __aexit__(self, *args):
await super().__aexit__(*args)
self._updateFuture.cancel()
def handleError(self, msg):
errorCode = int(msg)
if errorCode == 0: # no error
return
async def updateStatus(self):
while True:
await asyncio.sleep(0.2)
if (self.connection is None):
continue
try:
await self.singleUpdate()
except:
logging.error(traceback.format_exc())
async def singleUpdate(self):
movFut = self._isMovingFuture
self._status = (await self.send("?astat"))[self.axis - 1]
self.set_trait('statusMessage', self._StatusMap[self._status])
cnt = await self.queryAxisVariable('cnt') * ureg.count
with ureg.context(self.contextName):
cnt = cnt.to(self.prefPosUnit)
self.set_trait('value', cnt)
if self._status not in self._movingStates:
self.set_trait('status', self.Status.Idle)
if not movFut.done():
movFut.set_result(None)
else:
self.set_trait('status', self.Status.Moving)
errCode = await self.send("?err")
if (errCode != 0):
logging.error("OWIS Axis {} Error: {}".format(self.axis, errCode))
estat = await self.queryAxisVariable('estat')
self.set_trait('limitSwitchActive', bool(estat & 0xF))
self.set_trait('motorPowerStageError', bool(estat & (1 << 4)))
async def setAxisVariable(self, var, value):
cmd = '{}{}={}'.format(var, int(self.axis), value)
await self.send(cmd)
async def queryAxisVariable(self, var):
cmd = '?{}{}'.format(var, int(self.axis))
return await self.send(cmd)
async def send(self, command):
""" Send a command to the controller. If the command is a request,
the reply will be parsed (if possible) and returned. A error message
is automatically sent to check for communcation errors.
Parameters
----------
command (convertible to bytearray) : The command to be sent.
"""
if self.connection is None:
return None
ret = await self.connection.send(command)
if ret is not None:
try:
ret = int(ret)
except ValueError:
pass
errorCode = int(await self.connection.send('?msg'))
if errorCode != 0:
raise Exception("Message error code %d on OWIS Axis %d." %
(errorCode, self.axis))
return ret
async def moveTo(self, val: float, velocity=None):
if velocity is None:
velocity = self.velocity
with ureg.context(self.contextName):
velocity = velocity.to('count/s').magnitude
# From the manual. 256µs is the sampling period of the encoder
velocity *= 256e-6 * 65536
val = val.to('count').magnitude
await self.halt()
await self.singleUpdate()
self.set_trait('status', self.Status.Moving)
await self.setAxisVariable("pvel", int(velocity))
await self.setAxisVariable("pset", int(val))
await self.send("pgo" + str(self.axis))
if self._isMovingFuture.done():
self._isMovingFuture = asyncio.Future()
return await self._isMovingFuture
@action("Halt", priority=0)
async def halt(self):
if not self._isMovingFuture.done():
self._isMovingFuture.cancel()
await self.send('stop' + str(self.axis))
@action("Set Position to zero", priority=1)
async def resetCounter(self):
await self.send('cres' + str(self.axis))
@action("Free from limit switch", priority=2)
async def efree(self):
await self.send('efree' + str(self.axis))
@action("Home to min. lim. switch", priority=3)
async def homeMinLim(self):
await self.setAxisVariable('ref', 4)
@action("Initialize", priority=4)
async def inititialize(self):
await self.setAxisVariable('axis', 1)
await self.send('init' + str(self.axis))
@action("Release axis", priority=5)
async def releaseAxis(self):
await self.setAxisVariable('axis', 0)
@action("Unrelease axis", priority=6)
async def unreleaseAxis(self):
await self.setAxisVariable('axis', 1)
def stop(self):
self._loop.create_task(self.halt())
async def reference(self):
pass
class YAPInteractiveApp(Configurable):
"""A Mixin for applications that start YAPInteractive instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
The following methods should be called by the :meth:`initialize` method
of the subclass:
- :meth:`init_path`
- :meth:`init_shell` (to be implemented by the subclass)
- :meth:`init_gui_pylab`
- :meth:`init_extensions`
- :meth:`init_code`
"""
extensions = List(
Unicode(),
help="A list of dotted module names of IPython extensions to load."
).tag(config=True)
extra_extension = Unicode(
'', help="dotted module name of an IPython extension to load.").tag(
config=True)
reraise_ipython_extension_failures = Bool(
False,
help="Reraise exceptions encountered loading IPython extensions?",
).tag(config=True)
# Extensions that are always loaded (not configurable)
default_extensions = List(Unicode(), [u'storemagic']).tag(config=False)
hide_initial_ns = Bool(
True,
help=
"""Should variables loaded at startup (by startup files, exec_lines, etc.)
be hidden from tools like %who?""").tag(config=True)
exec_files = List(
Unicode(),
help="""List of files to run at IPython startup.""").tag(config=True)
exec_PYTHONSTARTUP = Bool(
True,
help="""Run the file referenced by the PYTHONSTARTUP environment
variable at IPython startup.""").tag(config=True)
file_to_run = Unicode('', help="""A file to be run""").tag(config=True)
exec_lines = List(
Unicode(),
help="""lines of code to run at IPython startup.""").tag(config=True)
code_to_run = Unicode(
'', help="Execute the given command string.").tag(config=True)
module_to_run = Unicode(
'', help="Run the module as a script.").tag(config=True)
gui = CaselessStrEnum(
gui_keys,
allow_none=True,
help="Enable GUI event loop integration with any of {0}.".format(
gui_keys)).tag(config=True)
matplotlib = CaselessStrEnum(
backend_keys,
allow_none=True,
help="""Configure matplotlib for interactive use with
the default matplotlib backend.""").tag(config=True)
pylab = CaselessStrEnum(
backend_keys,
allow_none=True,
help="""Pre-load matplotlib and numpy for interactive use,
selecting a particular matplotlib backend and loop integration.
""").tag(config=True)
pylab_import_all = Bool(
True,
help=
"""If true, IPython will populate the user namespace with numpy, pylab, etc.
and an ``import *`` is done from numpy and pylab, when using pylab mode.
When False, pylab mode should not import any names into the user namespace.
""").tag(config=True)
shell = Instance('yap_ipython.core.interactiveshell.YAPInteractiveABC',
allow_none=True)
# whether interact-loop should start
interact = Bool(True)
user_ns = Instance(dict, args=None, allow_none=True)
@observe('user_ns')
def _user_ns_changed(self, change):
if self.shell is not None:
self.shell.user_ns = change['new']
self.shell.init_user_ns()
def init_path(self):
"""Add current working directory, '', to sys.path"""
if sys.path[0] != '':
sys.path.insert(0, '')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
enable = False
shell = self.shell
if self.pylab:
enable = lambda key: shell.enable_pylab(
key, import_all=self.pylab_import_all)
key = self.pylab
elif self.matplotlib:
enable = shell.enable_matplotlib
key = self.matplotlib
elif self.gui:
enable = shell.enable_gui
key = self.gui
if not enable:
return
try:
r = enable(key)
except ImportError:
self.log.warning(
"Eventloop or matplotlib integration failed. Is matplotlib installed?"
)
self.shell.showtraceback()
return
except Exception:
self.log.warning("GUI event loop or pylab initialization failed")
self.shell.showtraceback()
return
if isinstance(r, tuple):
gui, backend = r[:2]
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s, matplotlib=%s", gui, backend)
if key == "auto":
print("Using matplotlib backend: %s" % backend)
else:
gui = r
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s", gui)
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
try:
self.log.debug("Loading IPython extensions...")
extensions = self.default_extensions + self.extensions
if self.extra_extension:
extensions.append(self.extra_extension)
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
if self.reraise_ipython_extension_failures:
raise
msg = ("Error in loading extension: {ext}\n"
"Check your config files in {location}".format(
ext=ext, location=self.profile_dir.location))
self.log.warning(msg, exc_info=True)
except:
if self.reraise_ipython_extension_failures:
raise
self.log.warning("Unknown error in loading extensions:",
exc_info=True)
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
# Hide variables defined here from %who etc.
if self.hide_initial_ns:
self.shell.user_ns_hidden.update(self.shell.user_ns)
# command-line execution (ipython -i script.py, ipython -m module)
# should *not* be excluded from %whos
self._run_cmd_line_code()
self._run_module()
# flush output, so itwon't be attached to the first cell
sys.stdout.flush()
sys.stderr.flush()
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" % line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warning("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warning(
"Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname, shell_futures=False):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError:
self.log.warning("File not found: %r" % fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
try:
if os.path.isfile(full_filename):
self.log.info("Running file in user namespace: %s" %
full_filename)
# Ensure that __file__ is always defined to match Python
# behavior.
with preserve_keys(self.shell.user_ns, '__file__'):
self.shell.user_ns['__file__'] = fname
if full_filename.endswith('.ipy'):
self.shell.safe_execfile_ipy(
full_filename, shell_futures=shell_futures)
else:
# default to python, even without extension
self.shell.safe_execfile(full_filename,
self.shell.user_ns,
shell_futures=shell_futures,
raise_exceptions=True)
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dirs = [self.profile_dir.startup_dir] + [
os.path.join(p, 'startup')
for p in chain(ENV_CONFIG_DIRS, SYSTEM_CONFIG_DIRS)
]
startup_files = []
if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \
not (self.file_to_run or self.code_to_run or self.module_to_run):
python_startup = os.environ['PYTHONSTARTUP']
self.log.debug("Running PYTHONSTARTUP file %s...", python_startup)
try:
self._exec_file(python_startup)
except:
self.log.warning(
"Unknown error in handling PYTHONSTARTUP file %s:",
python_startup)
self.shell.showtraceback()
for startup_dir in startup_dirs[::-1]:
startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warning("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warning(
"Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warning(
"Error in executing line in user namespace: %s" % line)
self.shell.showtraceback()
if not self.interact:
self.exit(1)
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
if os.path.isdir(fname):
fname = os.path.join(fname, "__main__.py")
try:
self._exec_file(fname, shell_futures=True)
except:
self.shell.showtraceback(tb_offset=4)
if not self.interact:
self.exit(1)
def _run_module(self):
"""Run module specified at the command-line."""
if self.module_to_run:
# Make sure that the module gets a proper sys.argv as if it were
# run using `python -m`.
save_argv = sys.argv
sys.argv = [sys.executable] + self.extra_args
try:
self.shell.safe_run_module(self.module_to_run,
self.shell.user_ns)
finally:
sys.argv = save_argv
class OSMagics(Magics):
"""Magics to interact with the underlying OS (shell-type functionality).
"""
cd_force_quiet = Bool(False,
help="Force %cd magic to be quiet even if -q is not passed."
).tag(config=True)
def __init__(self, shell=None, **kwargs):
# Now define isexec in a cross platform manner.
self.is_posix = False
self.execre = None
if os.name == 'posix':
self.is_posix = True
else:
try:
winext = os.environ['pathext'].replace(';','|').replace('.','')
except KeyError:
winext = 'exe|com|bat|py'
self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
# call up the chain
super().__init__(shell=shell, **kwargs)
@skip_doctest
def _isexec_POSIX(self, file):
"""
Test for executable on a POSIX system
"""
if os.access(file.path, os.X_OK):
# will fail on maxOS if access is not X_OK
return file.is_file()
return False
@skip_doctest
def _isexec_WIN(self, file):
"""
Test for executable file on non POSIX system
"""
return file.is_file() and self.execre.match(file.name) is not None
@skip_doctest
def isexec(self, file):
"""
Test for executable file on non POSIX system
"""
if self.is_posix:
return self._isexec_POSIX(file)
else:
return self._isexec_WIN(file)
@skip_doctest
@line_magic
def alias(self, parameter_s=''):
"""Define an alias for a system command.
'%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
Then, typing 'alias_name params' will execute the system command 'cmd
params' (from your underlying operating system).
Aliases have lower precedence than magic functions and Python normal
variables, so if 'foo' is both a Python variable and an alias, the
alias can not be executed until 'del foo' removes the Python variable.
You can use the %l specifier in an alias definition to represent the
whole line when the alias is called. For example::
In [2]: alias bracket echo "Input in brackets: <%l>"
In [3]: bracket hello world
Input in brackets: <hello world>
You can also define aliases with parameters using %s specifiers (one
per parameter)::
In [1]: alias parts echo first %s second %s
In [2]: %parts A B
first A second B
In [3]: %parts A
Incorrect number of arguments: 2 expected.
parts is an alias to: 'echo first %s second %s'
Note that %l and %s are mutually exclusive. You can only use one or
the other in your aliases.
Aliases expand Python variables just like system calls using ! or !!
do: all expressions prefixed with '$' get expanded. For details of
the semantic rules, see PEP-215:
http://www.python.org/peps/pep-0215.html. This is the library used by
IPython for variable expansion. If you want to access a true shell
variable, an extra $ is necessary to prevent its expansion by
IPython::
In [6]: alias show echo
In [7]: PATH='A Python string'
In [8]: show $PATH
A Python string
In [9]: show $$PATH
/usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
You can use the alias facility to access all of $PATH. See the %rehashx
function, which automatically creates aliases for the contents of your
$PATH.
If called with no parameters, %alias prints the current alias table
for your system. For posix systems, the default aliases are 'cat',
'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
aliases are added. For windows-based systems, the default aliases are
'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
You can see the definition of alias by adding a question mark in the
end::
In [1]: cat?
Repr: <alias cat for 'cat'>"""
par = parameter_s.strip()
if not par:
aliases = sorted(self.shell.alias_manager.aliases)
# stored = self.shell.db.get('stored_aliases', {} )
# for k, v in stored:
# atab.append(k, v[0])
print("Total number of aliases:", len(aliases))
sys.stdout.flush()
return aliases
# Now try to define a new one
try:
alias,cmd = par.split(None, 1)
except TypeError:
print(oinspect.getdoc(self.alias))
return
try:
self.shell.alias_manager.define_alias(alias, cmd)
except AliasError as e:
print(e)
# end magic_alias
@line_magic
def unalias(self, parameter_s=''):
"""Remove an alias"""
aname = parameter_s.strip()
try:
self.shell.alias_manager.undefine_alias(aname)
except ValueError as e:
print(e)
return
stored = self.shell.db.get('stored_aliases', {} )
if aname in stored:
print("Removing %stored alias",aname)
del stored[aname]
self.shell.db['stored_aliases'] = stored
@line_magic
def rehashx(self, parameter_s=''):
"""Update the alias table with all executable files in $PATH.
rehashx explicitly checks that every entry in $PATH is a file
with execute access (os.X_OK).
Under Windows, it checks executability as a match against a
'|'-separated string of extensions, stored in the IPython config
variable win_exec_ext. This defaults to 'exe|com|bat'.
This function also resets the root module cache of module completer,
used on slow filesystems.
"""
from IPython.core.alias import InvalidAliasError
# for the benefit of module completer in ipy_completers.py
del self.shell.db['rootmodules_cache']
path = [os.path.abspath(os.path.expanduser(p)) for p in
os.environ.get('PATH','').split(os.pathsep)]
syscmdlist = []
savedir = os.getcwd()
# Now walk the paths looking for executables to alias.
try:
# write the whole loop for posix/Windows so we don't have an if in
# the innermost part
if self.is_posix:
for pdir in path:
try:
os.chdir(pdir)
except OSError:
continue
# for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
dirlist = os.scandir(path=pdir)
for ff in dirlist:
if self.isexec(ff):
fname = ff.name
try:
# Removes dots from the name since ipython
# will assume names with dots to be python.
if not self.shell.alias_manager.is_alias(fname):
self.shell.alias_manager.define_alias(
fname.replace('.',''), fname)
except InvalidAliasError:
pass
else:
syscmdlist.append(fname)
else:
no_alias = Alias.blacklist
for pdir in path:
try:
os.chdir(pdir)
except OSError:
continue
# for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
dirlist = os.scandir(pdir)
for ff in dirlist:
fname = ff.name
base, ext = os.path.splitext(fname)
if self.isexec(ff) and base.lower() not in no_alias:
if ext.lower() == '.exe':
fname = base
try:
# Removes dots from the name since ipython
# will assume names with dots to be python.
self.shell.alias_manager.define_alias(
base.lower().replace('.',''), fname)
except InvalidAliasError:
pass
syscmdlist.append(fname)
self.shell.db['syscmdlist'] = syscmdlist
finally:
os.chdir(savedir)
@skip_doctest
@line_magic
def pwd(self, parameter_s=''):
"""Return the current working directory path.
Examples
--------
::
In [9]: pwd
Out[9]: '/home/tsuser/sprint/ipython'
"""
try:
return os.getcwd()
except FileNotFoundError:
raise UsageError("CWD no longer exists - please use %cd to change directory.")
@skip_doctest
@line_magic
def cd(self, parameter_s=''):
"""Change the current working directory.
This command automatically maintains an internal list of directories
you visit during your IPython session, in the variable _dh. The
command %dhist shows this history nicely formatted. You can also
do 'cd -<tab>' to see directory history conveniently.
Usage:
cd 'dir': changes to directory 'dir'.
cd -: changes to the last visited directory.
cd -<n>: changes to the n-th directory in the directory history.
cd --foo: change to directory that matches 'foo' in history
cd -b <bookmark_name>: jump to a bookmark set by %bookmark
(note: cd <bookmark_name> is enough if there is no
directory <bookmark_name>, but a bookmark with the name exists.)
'cd -b <tab>' allows you to tab-complete bookmark names.
Options:
-q: quiet. Do not print the working directory after the cd command is
executed. By default IPython's cd command does print this directory,
since the default prompts do not display path information.
Note that !cd doesn't work for this purpose because the shell where
!command runs is immediately discarded after executing 'command'.
Examples
--------
::
In [10]: cd parent/child
/home/tsuser/parent/child
"""
try:
oldcwd = os.getcwd()
except FileNotFoundError:
# Happens if the CWD has been deleted.
oldcwd = None
numcd = re.match(r'(-)(\d+)$',parameter_s)
# jump in directory history by number
if numcd:
nn = int(numcd.group(2))
try:
ps = self.shell.user_ns['_dh'][nn]
except IndexError:
print('The requested directory does not exist in history.')
return
else:
opts = {}
elif parameter_s.startswith('--'):
ps = None
fallback = None
pat = parameter_s[2:]
dh = self.shell.user_ns['_dh']
# first search only by basename (last component)
for ent in reversed(dh):
if pat in os.path.basename(ent) and os.path.isdir(ent):
ps = ent
break
if fallback is None and pat in ent and os.path.isdir(ent):
fallback = ent
# if we have no last part match, pick the first full path match
if ps is None:
ps = fallback
if ps is None:
print("No matching entry in directory history")
return
else:
opts = {}
else:
opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
# jump to previous
if ps == '-':
try:
ps = self.shell.user_ns['_dh'][-2]
except IndexError:
raise UsageError('%cd -: No previous directory to change to.')
# jump to bookmark if needed
else:
if not os.path.isdir(ps) or 'b' in opts:
bkms = self.shell.db.get('bookmarks', {})
if ps in bkms:
target = bkms[ps]
print('(bookmark:%s) -> %s' % (ps, target))
ps = target
else:
if 'b' in opts:
raise UsageError("Bookmark '%s' not found. "
"Use '%%bookmark -l' to see your bookmarks." % ps)
# at this point ps should point to the target dir
if ps:
try:
os.chdir(os.path.expanduser(ps))
if hasattr(self.shell, 'term_title') and self.shell.term_title:
set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
except OSError:
print(sys.exc_info()[1])
else:
cwd = os.getcwd()
dhist = self.shell.user_ns['_dh']
if oldcwd != cwd:
dhist.append(cwd)
self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
else:
os.chdir(self.shell.home_dir)
if hasattr(self.shell, 'term_title') and self.shell.term_title:
set_term_title(self.shell.term_title_format.format(cwd="~"))
cwd = os.getcwd()
dhist = self.shell.user_ns['_dh']
if oldcwd != cwd:
dhist.append(cwd)
self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
print(self.shell.user_ns['_dh'][-1])
@line_magic
def env(self, parameter_s=''):
"""Get, set, or list environment variables.
Usage:\\
%env: lists all environment variables/values
%env var: get value for var
%env var val: set value for var
%env var=val: set value for var
%env var=$val: set value for var, using python expansion if possible
"""
if parameter_s.strip():
split = '=' if '=' in parameter_s else ' '
bits = parameter_s.split(split)
if len(bits) == 1:
key = parameter_s.strip()
if key in os.environ:
return os.environ[key]
else:
err = "Environment does not have key: {0}".format(key)
raise UsageError(err)
if len(bits) > 1:
return self.set_env(parameter_s)
env = dict(os.environ)
# hide likely secrets when printing the whole environment
for key in list(env):
if any(s in key.lower() for s in ('key', 'token', 'secret')):
env[key] = '<hidden>'
return env
@line_magic
def set_env(self, parameter_s):
"""Set environment variables. Assumptions are that either "val" is a
name in the user namespace, or val is something that evaluates to a
string.
Usage:\\
%set_env var val: set value for var
%set_env var=val: set value for var
%set_env var=$val: set value for var, using python expansion if possible
"""
split = '=' if '=' in parameter_s else ' '
bits = parameter_s.split(split, 1)
if not parameter_s.strip() or len(bits)<2:
raise UsageError("usage is 'set_env var=val'")
var = bits[0].strip()
val = bits[1].strip()
if re.match(r'.*\s.*', var):
# an environment variable with whitespace is almost certainly
# not what the user intended. what's more likely is the wrong
# split was chosen, ie for "set_env cmd_args A=B", we chose
# '=' for the split and should have chosen ' '. to get around
# this, users should just assign directly to os.environ or use
# standard magic {var} expansion.
err = "refusing to set env var with whitespace: '{0}'"
err = err.format(val)
raise UsageError(err)
os.environ[var] = val
print('env: {0}={1}'.format(var,val))
@line_magic
def pushd(self, parameter_s=''):
"""Place the current dir on stack and change directory.
Usage:\\
%pushd ['dirname']
"""
dir_s = self.shell.dir_stack
tgt = os.path.expanduser(parameter_s)
cwd = os.getcwd().replace(self.shell.home_dir,'~')
if tgt:
self.cd(parameter_s)
dir_s.insert(0,cwd)
return self.shell.magic('dirs')
@line_magic
def popd(self, parameter_s=''):
"""Change to directory popped off the top of the stack.
"""
if not self.shell.dir_stack:
raise UsageError("%popd on empty stack")
top = self.shell.dir_stack.pop(0)
self.cd(top)
print("popd ->",top)
@line_magic
def dirs(self, parameter_s=''):
"""Return the current directory stack."""
return self.shell.dir_stack
@line_magic
def dhist(self, parameter_s=''):
"""Print your history of visited directories.
%dhist -> print full history\\
%dhist n -> print last n entries only\\
%dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
This history is automatically maintained by the %cd command, and
always available as the global list variable _dh. You can use %cd -<n>
to go to directory number <n>.
Note that most of time, you should view directory history by entering
cd -<TAB>.
"""
dh = self.shell.user_ns['_dh']
if parameter_s:
try:
args = map(int,parameter_s.split())
except:
self.arg_err(self.dhist)
return
if len(args) == 1:
ini,fin = max(len(dh)-(args[0]),0),len(dh)
elif len(args) == 2:
ini,fin = args
fin = min(fin, len(dh))
else:
self.arg_err(self.dhist)
return
else:
ini,fin = 0,len(dh)
print('Directory history (kept in _dh)')
for i in range(ini, fin):
print("%d: %s" % (i, dh[i]))
@skip_doctest
@line_magic
def sc(self, parameter_s=''):
"""Shell capture - run shell command and capture output (DEPRECATED use !).
DEPRECATED. Suboptimal, retained for backwards compatibility.
You should use the form 'var = !command' instead. Example:
"%sc -l myfiles = ls ~" should now be written as
"myfiles = !ls ~"
myfiles.s, myfiles.l and myfiles.n still apply as documented
below.
--
%sc [options] varname=command
IPython will run the given command using commands.getoutput(), and
will then update the user's interactive namespace with a variable
called varname, containing the value of the call. Your command can
contain shell wildcards, pipes, etc.
The '=' sign in the syntax is mandatory, and the variable name you
supply must follow Python's standard conventions for valid names.
(A special format without variable name exists for internal use)
Options:
-l: list output. Split the output on newlines into a list before
assigning it to the given variable. By default the output is stored
as a single string.
-v: verbose. Print the contents of the variable.
In most cases you should not need to split as a list, because the
returned value is a special type of string which can automatically
provide its contents either as a list (split on newlines) or as a
space-separated string. These are convenient, respectively, either
for sequential processing or to be passed to a shell command.
For example::
# Capture into variable a
In [1]: sc a=ls *py
# a is a string with embedded newlines
In [2]: a
Out[2]: 'setup.py\\nwin32_manual_post_install.py'
# which can be seen as a list:
In [3]: a.l
Out[3]: ['setup.py', 'win32_manual_post_install.py']
# or as a whitespace-separated string:
In [4]: a.s
Out[4]: 'setup.py win32_manual_post_install.py'
# a.s is useful to pass as a single command line:
In [5]: !wc -l $a.s
146 setup.py
130 win32_manual_post_install.py
276 total
# while the list form is useful to loop over:
In [6]: for f in a.l:
...: !wc -l $f
...:
146 setup.py
130 win32_manual_post_install.py
Similarly, the lists returned by the -l option are also special, in
the sense that you can equally invoke the .s attribute on them to
automatically get a whitespace-separated string from their contents::
In [7]: sc -l b=ls *py
In [8]: b
Out[8]: ['setup.py', 'win32_manual_post_install.py']
In [9]: b.s
Out[9]: 'setup.py win32_manual_post_install.py'
In summary, both the lists and strings used for output capture have
the following special attributes::
.l (or .list) : value as list.
.n (or .nlstr): value as newline-separated string.
.s (or .spstr): value as space-separated string.
"""
opts,args = self.parse_options(parameter_s, 'lv')
# Try to get a variable name and command to run
try:
# the variable name must be obtained from the parse_options
# output, which uses shlex.split to strip options out.
var,_ = args.split('=', 1)
var = var.strip()
# But the command has to be extracted from the original input
# parameter_s, not on what parse_options returns, to avoid the
# quote stripping which shlex.split performs on it.
_,cmd = parameter_s.split('=', 1)
except ValueError:
var,cmd = '',''
# If all looks ok, proceed
split = 'l' in opts
out = self.shell.getoutput(cmd, split=split)
if 'v' in opts:
print('%s ==\n%s' % (var, pformat(out)))
if var:
self.shell.user_ns.update({var:out})
else:
return out
@line_cell_magic
def sx(self, line='', cell=None):
"""Shell execute - run shell command and capture output (!! is short-hand).
%sx command
IPython will run the given command using commands.getoutput(), and
return the result formatted as a list (split on '\\n'). Since the
output is _returned_, it will be stored in ipython's regular output
cache Out[N] and in the '_N' automatic variables.
Notes:
1) If an input line begins with '!!', then %sx is automatically
invoked. That is, while::
!ls
causes ipython to simply issue system('ls'), typing::
!!ls
is a shorthand equivalent to::
%sx ls
2) %sx differs from %sc in that %sx automatically splits into a list,
like '%sc -l'. The reason for this is to make it as easy as possible
to process line-oriented shell output via further python commands.
%sc is meant to provide much finer control, but requires more
typing.
3) Just like %sc -l, this is a list with special attributes:
::
.l (or .list) : value as list.
.n (or .nlstr): value as newline-separated string.
.s (or .spstr): value as whitespace-separated string.
This is very useful when trying to use such lists as arguments to
system commands."""
if cell is None:
# line magic
return self.shell.getoutput(line)
else:
opts,args = self.parse_options(line, '', 'out=')
output = self.shell.getoutput(cell)
out_name = opts.get('out', opts.get('o'))
if out_name:
self.shell.user_ns[out_name] = output
else:
return output
system = line_cell_magic('system')(sx)
bang = cell_magic('!')(sx)
@line_magic
def bookmark(self, parameter_s=''):
"""Manage IPython's bookmark system.
%bookmark <name> - set bookmark to current dir
%bookmark <name> <dir> - set bookmark to <dir>
%bookmark -l - list all bookmarks
%bookmark -d <name> - remove bookmark
%bookmark -r - remove all bookmarks
You can later on access a bookmarked folder with::
%cd -b <name>
or simply '%cd <name>' if there is no directory called <name> AND
there is such a bookmark defined.
Your bookmarks persist through IPython sessions, but they are
associated with each profile."""
opts,args = self.parse_options(parameter_s,'drl',mode='list')
if len(args) > 2:
raise UsageError("%bookmark: too many arguments")
bkms = self.shell.db.get('bookmarks',{})
if 'd' in opts:
try:
todel = args[0]
except IndexError:
raise UsageError(
"%bookmark -d: must provide a bookmark to delete")
else:
try:
del bkms[todel]
except KeyError:
raise UsageError(
"%%bookmark -d: Can't delete bookmark '%s'" % todel)
elif 'r' in opts:
bkms = {}
elif 'l' in opts:
bks = sorted(bkms)
if bks:
size = max(map(len, bks))
else:
size = 0
fmt = '%-'+str(size)+'s -> %s'
print('Current bookmarks:')
for bk in bks:
print(fmt % (bk, bkms[bk]))
else:
if not args:
raise UsageError("%bookmark: You must specify the bookmark name")
elif len(args)==1:
bkms[args[0]] = os.getcwd()
elif len(args)==2:
bkms[args[0]] = args[1]
self.shell.db['bookmarks'] = bkms
@line_magic
def pycat(self, parameter_s=''):
"""Show a syntax-highlighted file through a pager.
This magic is similar to the cat utility, but it will assume the file
to be Python source and will show it with syntax highlighting.
This magic command can either take a local filename, an url,
an history range (see %history) or a macro as argument ::
%pycat myscript.py
%pycat 7-27
%pycat myMacro
%pycat http://www.example.com/myscript.py
"""
if not parameter_s:
raise UsageError('Missing filename, URL, input history range, '
'or macro.')
try :
cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
except (ValueError, IOError):
print("Error: no such file, variable, URL, history range or macro")
return
page.page(self.shell.pycolorize(source_to_unicode(cont)))
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'-a', '--append', action='store_true', default=False,
help='Append contents of the cell to an existing file. '
'The file will be created if it does not exist.'
)
@magic_arguments.argument(
'filename', type=str,
help='file to write'
)
@cell_magic
def writefile(self, line, cell):
"""Write the contents of the cell to a file.
The file will be overwritten unless the -a (--append) flag is specified.
"""
args = magic_arguments.parse_argstring(self.writefile, line)
if re.match(r'^(\'.*\')|(".*")$', args.filename):
filename = os.path.expanduser(args.filename[1:-1])
else:
filename = os.path.expanduser(args.filename)
if os.path.exists(filename):
if args.append:
print("Appending to %s" % filename)
else:
print("Overwriting %s" % filename)
else:
print("Writing %s" % filename)
mode = 'a' if args.append else 'w'
with io.open(filename, mode, encoding='utf-8') as f:
f.write(cell)
class FileContentsManager(FileManagerMixin, ContentsManager):
root_dir = Unicode(config=True)
@default('root_dir')
def _default_root_dir(self):
try:
return self.parent.notebook_dir
except AttributeError:
return getcwd()
save_script = Bool(
False,
config=True,
help='DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0')
@observe('save_script')
def _update_save_script(self):
self.log.warning("""
`--script` is deprecated and will be removed in notebook 5.0.
You can trigger nbconvert via pre- or post-save hooks:
ContentsManager.pre_save_hook
FileContentsManager.post_save_hook
A post-save hook has been registered that calls:
jupyter nbconvert --to script [notebook]
which behaves similarly to `--script`.
""")
self.post_save_hook = _post_save_script
post_save_hook = Any(None,
config=True,
help="""Python callable or importstring thereof
to be called on the path of a file just saved.
This can be used to process the file on disk,
such as converting the notebook to a script or HTML via nbconvert.
It will be called as (all arguments passed by keyword)::
hook(os_path=os_path, model=model, contents_manager=instance)
- path: the filesystem path to the file just written
- model: the model representing the file
- contents_manager: this ContentsManager instance
""")
@validate('post_save_hook')
def _validate_post_save_hook(self, proposal):
value = proposal['value']
if isinstance(value, string_types):
value = import_item(value)
if not callable(value):
raise TraitError("post_save_hook must be callable")
return value
def run_post_save_hook(self, model, os_path):
"""Run the post-save hook if defined, and log errors"""
if self.post_save_hook:
try:
self.log.debug("Running post-save hook on %s", os_path)
self.post_save_hook(os_path=os_path,
model=model,
contents_manager=self)
except Exception as e:
self.log.error("Post-save hook failed o-n %s",
os_path,
exc_info=True)
raise web.HTTPError(
500,
u'Unexpected error while running post hook save: %s' % e)
@validate('root_dir')
def _validate_root_dir(self, proposal):
"""Do a bit of validation of the root_dir."""
value = proposal['value']
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("%r is not a directory" % value)
return value
def _checkpoints_class_default(self):
return FileCheckpoints
def is_hidden(self, path):
"""Does the API style path correspond to a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
hidden : bool
Whether the path exists and is hidden.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def file_exists(self, path):
"""Returns True if the file exists, else returns False.
API-style wrapper for os.path.isfile
Parameters
----------
path : string
The relative path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the file exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path)
return os.path.isfile(os_path)
def dir_exists(self, path):
"""Does the API-style path refer to an extant directory?
API-style wrapper for os.path.isdir
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.isdir(os_path)
def exists(self, path):
"""Returns True if the path exists, else returns False.
API-style wrapper for os.path.exists
Parameters
----------
path : string
The API path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the target exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.exists(os_path)
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir):
self.log.info(
"Refusing to serve hidden directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
for name in os.listdir(os_dir):
try:
os_path = os.path.join(os_dir, name)
except UnicodeDecodeError as e:
self.log.warning("failed to decode filename '%s': %s",
name, e)
continue
# skip over broken symlinks in listing
if not os.path.exists(os_path):
self.log.warning("%s doesn't exist", os_path)
continue
elif not os.path.isfile(os_path) and not os.path.isdir(
os_path):
self.log.debug("%s not a regular file", os_path)
continue
if self.should_list(name) and not is_hidden(
os_path, self.root_dir):
contents.append(
self.get(path='%s/%s' % (path, name), content=False))
model['format'] = 'json'
return model
def _file_model(self, path, content=True, format=None):
"""Build a model for a file
if content is requested, include the file contents.
format:
If 'text', the contents will be decoded as UTF-8.
If 'base64', the raw bytes contents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
model = self._base_model(path)
model['type'] = 'file'
os_path = self._get_os_path(path)
model['mimetype'] = mimetypes.guess_type(os_path)[0]
if content:
content, format = self._read_file(os_path, format)
if model['mimetype'] is None:
default_mime = {
'text': 'text/plain',
'base64': 'application/octet-stream'
}[format]
model['mimetype'] = default_mime
model.update(
content=content,
format=format,
)
return model
def _notebook_model(self, path, content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
os_path = self._get_os_path(path)
nb = self._read_notebook(os_path, as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def get(self, path, content=True, type=None, format=None):
""" Takes a path for an entity and returns its model
Parameters
----------
path : str
the API path that describes the relative path for the target
content : bool
Whether to include the contents in the reply
type : str, optional
The requested type - 'file', 'notebook', or 'directory'.
Will raise HTTPError 400 if the content doesn't match.
format : str, optional
The requested format for file contents. 'text' or 'base64'.
Ignored if this returns a notebook or directory model.
Returns
-------
model : dict
the contents model. If content=True, returns the contents
of the file or directory as well.
"""
path = path.strip('/')
if not self.exists(path):
raise web.HTTPError(404, u'No such file or directory: %s' % path)
os_path = self._get_os_path(path)
if os.path.isdir(os_path):
if type not in (None, 'directory'):
raise web.HTTPError(400,
u'%s is a directory, not a %s' %
(path, type),
reason='bad type')
model = self._dir_model(path, content=content)
elif type == 'notebook' or (type is None and path.endswith('.ipynb')):
model = self._notebook_model(path, content=content)
else:
if type == 'directory':
raise web.HTTPError(400,
u'%s is not a directory' % path,
reason='bad type')
model = self._file_model(path, content=content, format=format)
return model
def _save_directory(self, os_path, model, path=''):
"""create a directory"""
if is_hidden(os_path, self.root_dir):
raise web.HTTPError(400,
u'Cannot create hidden directory %r' % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
os.mkdir(os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def save(self, model, path=''):
"""Save the file model and return the model with no content."""
path = path.strip('/')
if 'type' not in model:
raise web.HTTPError(400, u'No file type provided')
if 'content' not in model and model['type'] != 'directory':
raise web.HTTPError(400, u'No file content provided')
os_path = self._get_os_path(path)
self.log.debug("Saving %s", os_path)
self.run_pre_save_hook(model=model, path=path)
try:
if model['type'] == 'notebook':
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
self._save_notebook(os_path, nb)
# One checkpoint should always exist for notebooks.
if not self.checkpoints.list_checkpoints(path):
self.create_checkpoint(path)
elif model['type'] == 'file':
# Missing format will be handled internally by _save_file.
self._save_file(os_path, model['content'], model.get('format'))
elif model['type'] == 'directory':
self._save_directory(os_path, model, path)
else:
raise web.HTTPError(
400, "Unhandled contents type: %s" % model['type'])
except web.HTTPError:
raise
except Exception as e:
self.log.error(u'Error while saving file: %s %s',
path,
e,
exc_info=True)
raise web.HTTPError(
500, u'Unexpected error while saving file: %s %s' % (path, e))
validation_message = None
if model['type'] == 'notebook':
self.validate_notebook_model(model)
validation_message = model.get('message', None)
model = self.get(path, content=False)
if validation_message:
model['message'] = validation_message
self.run_post_save_hook(model=model, os_path=os_path)
return model
def delete_file(self, path):
"""Delete file at path."""
path = path.strip('/')
os_path = self._get_os_path(path)
rm = os.unlink
if os.path.isdir(os_path):
listing = os.listdir(os_path)
# Don't delete non-empty directories.
# A directory containing only leftover checkpoints is
# considered empty.
cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None)
for entry in listing:
if entry != cp_dir:
raise web.HTTPError(400,
u'Directory %s not empty' % os_path)
elif not os.path.isfile(os_path):
raise web.HTTPError(404, u'File does not exist: %s' % os_path)
if os.path.isdir(os_path):
self.log.debug("Removing directory %s", os_path)
with self.perm_to_403():
shutil.rmtree(os_path)
else:
self.log.debug("Unlinking file %s", os_path)
with self.perm_to_403():
rm(os_path)
def rename_file(self, old_path, new_path):
"""Rename a file."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_path == old_path:
return
new_os_path = self._get_os_path(new_path)
old_os_path = self._get_os_path(old_path)
# Should we proceed with the move?
if os.path.exists(new_os_path) and not samefile(
old_os_path, new_os_path):
raise web.HTTPError(409, u'File already exists: %s' % new_path)
# Move the file
try:
with self.perm_to_403():
shutil.move(old_os_path, new_os_path)
except web.HTTPError:
raise
except Exception as e:
raise web.HTTPError(
500, u'Unknown error renaming file: %s %s' % (old_path, e))
def info_string(self):
return "Serving notebooks from local directory: %s" % self.root_dir
def get_kernel_path(self, path, model=None):
"""Return the initial API path of a kernel associated with a given notebook"""
if self.dir_exists(path):
return path
if '/' in path:
parent_dir = path.rsplit('/', 1)[0]
else:
parent_dir = ''
return parent_dir
class BaseIPythonApplication(Application):
name = Unicode(u'ipython')
description = Unicode(u'IPython: an enhanced interactive Python shell.')
version = Unicode(release.version)
aliases = Dict(base_aliases)
flags = Dict(base_flags)
classes = List([ProfileDir])
# enable `load_subconfig('cfg.py', profile='name')`
python_config_loader_class = ProfileAwareConfigLoader
# Track whether the config_file has changed,
# because some logic happens only if we aren't using the default.
config_file_specified = Set()
config_file_name = Unicode()
def _config_file_name_default(self):
return self.name.replace('-', '_') + u'_config.py'
def _config_file_name_changed(self, name, old, new):
if new != old:
self.config_file_specified.add(new)
# The directory that contains IPython's builtin profiles.
builtin_profile_dir = Unicode(
os.path.join(get_ipython_package_dir(), u'config', u'profile',
u'default'))
config_file_paths = List(Unicode())
def _config_file_paths_default(self):
return [py3compat.getcwd()]
extra_config_file = Unicode(config=True,
help="""Path to an extra config file to load.
If specified, load this config file in addition to any other IPython config.
""")
def _extra_config_file_changed(self, name, old, new):
try:
self.config_files.remove(old)
except ValueError:
pass
self.config_file_specified.add(new)
self.config_files.append(new)
profile = Unicode(u'default',
config=True,
help="""The IPython profile to use.""")
def _profile_changed(self, name, old, new):
self.builtin_profile_dir = os.path.join(get_ipython_package_dir(),
u'config', u'profile', new)
ipython_dir = Unicode(config=True,
help="""
The name of the IPython directory. This directory is used for logging
configuration (through profiles), history storage, etc. The default
is usually $HOME/.ipython. This option can also be specified through
the environment variable IPYTHONDIR.
""")
def _ipython_dir_default(self):
d = get_ipython_dir()
self._ipython_dir_changed('ipython_dir', d, d)
return d
_in_init_profile_dir = False
profile_dir = Instance(ProfileDir, allow_none=True)
def _profile_dir_default(self):
# avoid recursion
if self._in_init_profile_dir:
return
# profile_dir requested early, force initialization
self.init_profile_dir()
return self.profile_dir
overwrite = Bool(
False,
config=True,
help="""Whether to overwrite existing config files when copying""")
auto_create = Bool(
False,
config=True,
help="""Whether to create profile dir if it doesn't exist""")
config_files = List(Unicode())
def _config_files_default(self):
return [self.config_file_name]
copy_config_files = Bool(
False,
config=True,
help="""Whether to install the default config files into the profile dir.
If a new profile is being created, and IPython contains config files for that
profile, then they will be staged into the new directory. Otherwise,
default config files will be automatically generated.
""")
verbose_crash = Bool(
False,
config=True,
help=
"""Create a massive crash report when IPython encounters what may be an
internal error. The default is to append a short message to the
usual traceback""")
# The class to use as the crash handler.
crash_handler_class = Type(crashhandler.CrashHandler)
@catch_config_error
def __init__(self, **kwargs):
super(BaseIPythonApplication, self).__init__(**kwargs)
# ensure current working directory exists
try:
py3compat.getcwd()
except:
# exit if cwd doesn't exist
self.log.error("Current working directory doesn't exist.")
self.exit(1)
#-------------------------------------------------------------------------
# Various stages of Application creation
#-------------------------------------------------------------------------
def initialize_subcommand(self, subc, argv=None):
if subc in self.deprecated_subcommands:
import time
self.log.warning(
"Subcommand `ipython {sub}` is deprecated and will be removed "
"in future versions.".format(sub=subc))
self.log.warning(
"You likely want to use `jupyter {sub}`... continue "
"in 5 sec. Press Ctrl-C to quit now.".format(sub=subc))
try:
time.sleep(5)
except KeyboardInterrupt:
sys.exit(1)
return super(BaseIPythonApplication,
self).initialize_subcommand(subc, argv)
def init_crash_handler(self):
"""Create a crash handler, typically setting sys.excepthook to it."""
self.crash_handler = self.crash_handler_class(self)
sys.excepthook = self.excepthook
def unset_crashhandler():
sys.excepthook = sys.__excepthook__
atexit.register(unset_crashhandler)
def excepthook(self, etype, evalue, tb):
"""this is sys.excepthook after init_crashhandler
set self.verbose_crash=True to use our full crashhandler, instead of
a regular traceback with a short message (crash_handler_lite)
"""
if self.verbose_crash:
return self.crash_handler(etype, evalue, tb)
else:
return crashhandler.crash_handler_lite(etype, evalue, tb)
def _ipython_dir_changed(self, name, old, new):
if old is not Undefined:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding())
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding())
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(get_ipython_package_dir(), u'config',
u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def load_config_file(self, suppress_errors=True):
"""Load the config file.
By default, errors in loading config are handled, and a warning
printed on screen. For testing, the suppress_errors option is set
to False, so errors will make tests fail.
"""
self.log.debug("Searching path %s for config files",
self.config_file_paths)
base_config = 'ipython_config.py'
self.log.debug("Attempting to load config file: %s" % base_config)
try:
Application.load_config_file(self,
base_config,
path=self.config_file_paths)
except ConfigFileNotFound:
# ignore errors loading parent
self.log.debug("Config file %s not found", base_config)
pass
for config_file_name in self.config_files:
if not config_file_name or config_file_name == base_config:
continue
self.log.debug("Attempting to load config file: %s" %
self.config_file_name)
try:
Application.load_config_file(self,
config_file_name,
path=self.config_file_paths)
except ConfigFileNotFound:
# Only warn if the default config file was NOT being used.
if config_file_name in self.config_file_specified:
msg = self.log.warning
else:
msg = self.log.debug
msg("Config file not found, skipping: %s", config_file_name)
except Exception:
# For testing purposes.
if not suppress_errors:
raise
self.log.warning("Error loading config file: %s" %
self.config_file_name,
exc_info=True)
def init_profile_dir(self):
"""initialize the profile dir"""
self._in_init_profile_dir = True
if self.profile_dir is not None:
# already ran
return
if 'ProfileDir.location' not in self.config:
# location not specified, find by profile name
try:
p = ProfileDir.find_profile_dir_by_name(
self.ipython_dir, self.profile, self.config)
except ProfileDirError:
# not found, maybe create it (always create default profile)
if self.auto_create or self.profile == 'default':
try:
p = ProfileDir.create_profile_dir_by_name(
self.ipython_dir, self.profile, self.config)
except ProfileDirError:
self.log.fatal("Could not create profile: %r" %
self.profile)
self.exit(1)
else:
self.log.info("Created profile dir: %r" % p.location)
else:
self.log.fatal("Profile %r not found." % self.profile)
self.exit(1)
else:
self.log.debug("Using existing profile dir: %r" % p.location)
else:
location = self.config.ProfileDir.location
# location is fully specified
try:
p = ProfileDir.find_profile_dir(location, self.config)
except ProfileDirError:
# not found, maybe create it
if self.auto_create:
try:
p = ProfileDir.create_profile_dir(
location, self.config)
except ProfileDirError:
self.log.fatal(
"Could not create profile directory: %r" %
location)
self.exit(1)
else:
self.log.debug("Creating new profile dir: %r" %
location)
else:
self.log.fatal("Profile directory %r not found." %
location)
self.exit(1)
else:
self.log.info("Using existing profile dir: %r" % location)
# if profile_dir is specified explicitly, set profile name
dir_name = os.path.basename(p.location)
if dir_name.startswith('profile_'):
self.profile = dir_name[8:]
self.profile_dir = p
self.config_file_paths.append(p.location)
self._in_init_profile_dir = False
def init_config_files(self):
"""[optionally] copy default config files into profile dir."""
self.config_file_paths.extend(SYSTEM_CONFIG_DIRS)
# copy config files
path = self.builtin_profile_dir
if self.copy_config_files:
src = self.profile
cfg = self.config_file_name
if path and os.path.exists(os.path.join(path, cfg)):
self.log.warning(
"Staging %r from %s into %r [overwrite=%s]" %
(cfg, src, self.profile_dir.location, self.overwrite))
self.profile_dir.copy_config_file(cfg,
path=path,
overwrite=self.overwrite)
else:
self.stage_default_config_file()
else:
# Still stage *bundled* config files, but not generated ones
# This is necessary for `ipython profile=sympy` to load the profile
# on the first go
files = glob.glob(os.path.join(path, '*.py'))
for fullpath in files:
cfg = os.path.basename(fullpath)
if self.profile_dir.copy_config_file(cfg,
path=path,
overwrite=False):
# file was copied
self.log.warning(
"Staging bundled %s from %s into %r" %
(cfg, self.profile, self.profile_dir.location))
def stage_default_config_file(self):
"""auto generate default config file, and stage it into the profile."""
s = self.generate_config_file()
fname = os.path.join(self.profile_dir.location, self.config_file_name)
if self.overwrite or not os.path.exists(fname):
self.log.warning("Generating default config file: %r" % (fname))
with open(fname, 'w') as f:
f.write(s)
@catch_config_error
def initialize(self, argv=None):
# don't hook up crash handler before parsing command-line
self.parse_command_line(argv)
self.init_crash_handler()
if self.subapp is not None:
# stop here if subapp is taking over
return
cl_config = self.config
self.init_profile_dir()
self.init_config_files()
self.load_config_file()
# enforce cl-opts override configfile opts:
self.update_config(cl_config)
class Innotater(VBox): #VBox
_view_name = Unicode('InnotaterView').tag(sync=True)
_model_name = Unicode('InnotaterModel').tag(sync=True)
_view_module = Unicode('jupyter-innotater').tag(sync=True)
_model_module = Unicode('jupyter-innotater').tag(sync=True)
_view_module_version = Unicode(semver_range).tag(sync=True)
_model_module_version = Unicode(semver_range).tag(sync=True)
index = Int().tag(sync=True)
keyboard_shortcuts = Bool(False).tag(sync=True)
is_dirty = Bool(False).tag(sync=True)
def __init__(self,
inputs,
targets,
indexes=None,
keyboard_shortcuts=True,
save_hook=None,
vertical=False):
self.path = ''
self.dirty_uindexes = set()
self.save_hook = save_hook
self.datamanager = DataManager(inputs, targets, indexes)
slider = IntSlider(min=0, max=0)
self.slider = slider
self.prevbtn = Button(description='< Previous')
self.nextbtn = Button(description='Next >')
self.input_widgets = [
dw.get_widget() for dw in self.datamanager.get_inputs()
]
self.target_widgets = [
dw.get_widget() for dw in self.datamanager.get_targets()
]
self.add_class('innotater-base')
cbar_widgets = [self.prevbtn, slider, self.nextbtn]
if self.save_hook:
self.savebtn = Button(description='Save', disabled=True)
cbar_widgets.append(self.savebtn)
controlbar_widget = HBox(cbar_widgets)
controlbar_widget.add_class('innotater-controlbar')
InnotaterBox = HBox
if vertical:
InnotaterBox = VBox
self.add_class('innotater-base-vertical')
super().__init__([
InnotaterBox([VBox(self.input_widgets),
VBox(self.target_widgets)]), controlbar_widget
])
widgets.jslink((slider, 'value'), (self, 'index'))
self._observe_targets(self.datamanager.get_targets())
for dw in list(self.datamanager.get_all()):
dw.post_widget_create(self.datamanager)
self.prevbtn.on_click(lambda c: self.move_slider(-1))
self.nextbtn.on_click(lambda c: self.move_slider(1))
if self.save_hook:
self.savebtn.on_click(lambda c: self.save_hook_fire())
self.slider.max = self.datamanager.get_data_len() - 1
self.index = 0
self.keyboard_shortcuts = keyboard_shortcuts
self.on_msg(self.handle_message)
self.suspend_observed_changes = False
self.update_ui()
@observe('index')
def slider_changed(self, change):
self.update_ui()
def move_slider(self, change):
if change < 0 < self.index:
self.index -= 1
elif change > 0 and self.index < self.datamanager.get_data_len() - 1:
self.index += 1
def handle_message(self, _, content, buffers):
if content['event'] == 'keydown':
code = content['code']
self.handle_keypress(code)
def handle_keypress(self, code):
if self.suspend_observed_changes:
return
if code == 78: # n was 110
self.move_slider(1)
elif code == 80: # p was 112
self.move_slider(-1)
elif code == 83: # s
self.save_hook_fire()
def update_ui(self):
uindex = self.datamanager.get_underlying_index(self.index)
self.suspend_observed_changes = True
for dw in self.datamanager.get_all():
dw.update_ui(uindex)
self.suspend_observed_changes = False
self.prevbtn.disabled = self.index <= 0
self.nextbtn.disabled = self.index >= self.datamanager.get_data_len(
) - 1
def update_data(self, change):
if self.suspend_observed_changes:
return
uindex = self.datamanager.get_underlying_index(self.index)
# Find the Innotation that contains the widget that observed the change
widg = change['owner']
for dw in self.datamanager.get_targets():
if dw.contains_widget(widg):
dw.update_data(uindex)
self._make_dirty(uindex)
def _make_dirty(self, uindex):
self.dirty_uindexes.add(uindex)
if self.save_hook and not self.is_dirty:
self.is_dirty = True
self.savebtn.disabled = not self.is_dirty
def save_hook_fire(self):
if self.save_hook:
self.savebtn.disabled = True # Disable during save
has_saved = self.save_hook(list(self.dirty_uindexes))
if has_saved:
self.is_dirty = False
self.dirty_uindexes.clear()
self.savebtn.disabled = not self.is_dirty
def add_innotations(self, inputs, targets):
self.datamanager.dynamic_add_innotations(inputs, targets)
self._observe_targets(targets)
for dw in inputs + targets:
dw.post_widget_create(self.datamanager)
def _observe_targets(self, targets):
for dw in targets:
dw.widget_observe(self.update_data, names='value')
if isinstance(dw, ChildrenChangeNotifierMixin):
dw.on_children_changed(self.new_children_handler)
if isinstance(dw, DataChangeNotifierMixin):
dw.on_data_changed(self.updated_data_handler)
def new_children_handler(self, parent, newchildren):
self.add_innotations([], newchildren) # Assume always targets
self.update_ui()
def updated_data_handler(self, widget):
uindex = self.datamanager.get_underlying_index(self.index)
self._make_dirty(uindex)
self.update_ui()
class DetailsView(RegulusDOMWidget):
""""""
_model_name = Unicode('DetailsModel').tag(sync=True)
_view_name = Unicode('DetailsView').tag(sync=True)
title = Unicode('title').tag(sync=True)
data = Instance(klass=DataWidget).tag(sync=True, **widget_serialization)
measure = Unicode(None, allow_none=True).tag(sync=True)
tree_model = Instance(klass=TreeWidget,
allow_none=True).tag(sync=True,
**widget_serialization)
show = List().tag(sync=True)
highlight = Int(-2).tag(sync=True)
show_inverse = Bool(True).tag(sync=True)
inverse = Dict(allow_none=True).tag(sync=True)
cmap = Unicode('RdYlBu').tag(sync=True)
def __init__(self, **kwargs):
self._inverse_cache = set()
if 'data' in kwargs:
data = kwargs['data']
if not isinstance(data, DataWidget):
data = DataWidget(data=data)
kwargs['data'] = data
if 'measure' not in kwargs:
kwargs['measure'] = data.data.measure
super().__init__(**kwargs)
def reset_inverse(self):
self._inverse_cache.clear()
self._show({'new': self.show})
def _send_msg(self, pid, data):
self.inverse = {pid: data}
self.inverse = None
@observe('show')
def _show(self, change):
self._update_inverse()
@observe('show_inverse')
def _show_inverse(self, change):
self._update_inverse()
def _update_inverse(self):
r = self.data.data
if self.show_inverse and self.data is not None:
if not self.data.data.attr.has('inverse_regression'):
self.data.data.add_attr(default_inverse_regression,
name='inverse_regression')
pids = filter(lambda pid: pid not in self._inverse_cache,
self.show)
msg = {}
t0 = time()
for node in r.find_nodes(pids):
line = r.attr['inverse_regression'][node]
msg[node.id] = convert(line)
self._inverse_cache.add(node.id)
if time() - t0 > 1:
self.inverse = msg
self.inverse = None
msg = {}
t0 = time()
if len(msg) > 0:
self.inverse = msg
self.inverse = None
class NBExtensionApp(JupyterApp):
"""Entry point for installing notebook extensions"""
description = """Install Jupyter notebook extensions
Usage
jupyter install-nbextension path/url
This copies a file or a folder into the Jupyter nbextensions directory.
If a URL is given, it will be downloaded.
If an archive is given, it will be extracted into nbextensions.
If the requested files are already up to date, no action is taken
unless --overwrite is specified.
"""
examples = """
jupyter install-nbextension /path/to/myextension
"""
aliases = aliases
flags = flags
overwrite = Bool(False,
config=True,
help="Force overwrite of existing files")
symlink = Bool(False,
config=True,
help="Create symlinks instead of copying files")
user = Bool(False, config=True, help="Whether to do a user install")
prefix = Unicode('', config=True, help="Installation prefix")
nbextensions_dir = Unicode(
'',
config=True,
help="Full path to nbextensions dir (probably use prefix or user)")
destination = Unicode('',
config=True,
help="Destination for the copy or symlink")
verbose = Enum((0, 1, 2),
default_value=1,
config=True,
help="Verbosity level")
def install_extensions(self):
if len(self.extra_args) > 1:
raise ValueError(
"only one nbextension allowed at a time. Call multiple times to install multiple extensions."
)
install_nbextension(
self.extra_args[0],
overwrite=self.overwrite,
symlink=self.symlink,
verbose=self.verbose,
user=self.user,
prefix=self.prefix,
destination=self.destination,
nbextensions_dir=self.nbextensions_dir,
)
def start(self):
if not self.extra_args:
for nbext in [pjoin(self.data_dir, u'nbextensions')
] + SYSTEM_NBEXTENSIONS_DIRS:
if os.path.exists(nbext):
print("Notebook extensions in %s:" % nbext)
for ext in os.listdir(nbext):
print(u" %s" % ext)
else:
try:
self.install_extensions()
except ArgumentConflict as e:
print(str(e), file=sys.stderr)
self.exit(1)
class IPythonWidget(FrontendWidget):
""" A FrontendWidget for an IPython kernel.
"""
# If set, the 'custom_edit_requested(str, int)' signal will be emitted when
# an editor is needed for a file. This overrides 'editor' and 'editor_line'
# settings.
custom_edit = Bool(False)
custom_edit_requested = QtCore.Signal(object, object)
editor = Unicode(default_editor,
config=True,
help="""
A command for invoking a system text editor. If the string contains a
{filename} format specifier, it will be used. Otherwise, the filename
will be appended to the end the command.
""")
editor_line = Unicode(config=True,
help="""
The editor command to use when a specific line number is requested. The
string should contain two format specifiers: {line} and {filename}. If
this parameter is not specified, the line number option to the %edit
magic will be ignored.
""")
style_sheet = Unicode(config=True,
help="""
A CSS stylesheet. The stylesheet can contain classes for:
1. Qt: QPlainTextEdit, QFrame, QWidget, etc
2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter)
3. IPython: .error, .in-prompt, .out-prompt, etc
""")
syntax_style = Unicode(config=True,
help="""
If not empty, use this Pygments style for syntax highlighting.
Otherwise, the style sheet is queried for Pygments style
information.
""")
# Prompts.
in_prompt = Unicode(default_in_prompt, config=True)
out_prompt = Unicode(default_out_prompt, config=True)
input_sep = Unicode(default_input_sep, config=True)
output_sep = Unicode(default_output_sep, config=True)
output_sep2 = Unicode(default_output_sep2, config=True)
# FrontendWidget protected class variables.
_input_splitter_class = IPythonInputSplitter
_prompt_transformer = IPythonInputSplitter(
physical_line_transforms=[ipy_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
# IPythonWidget protected class variables.
_PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number'])
_payload_source_edit = 'edit_magic'
_payload_source_exit = 'ask_exit'
_payload_source_next_input = 'set_next_input'
_payload_source_page = 'page'
_retrying_history_request = False
_starting = False
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(IPythonWidget, self).__init__(*args, **kw)
# IPythonWidget protected variables.
self._payload_handlers = {
self._payload_source_edit: self._handle_payload_edit,
self._payload_source_exit: self._handle_payload_exit,
self._payload_source_page: self._handle_payload_page,
self._payload_source_next_input: self._handle_payload_next_input
}
self._previous_prompt_obj = None
self._keep_kernel_on_exit = None
# Initialize widget styling.
if self.style_sheet:
self._style_sheet_changed()
self._syntax_style_changed()
else:
self.set_default_style()
self._guiref_loaded = False
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Reimplemented to support IPython's improved completion machinery.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
content = rep['content']
matches = content['matches']
start = content['cursor_start']
end = content['cursor_end']
start = max(start, 0)
end = max(end, start)
# Move the control's cursor to the desired end point
cursor_pos = self._get_input_buffer_cursor_pos()
if end < cursor_pos:
cursor.movePosition(QtGui.QTextCursor.Left,
n=(cursor_pos - end))
elif end > cursor_pos:
cursor.movePosition(QtGui.QTextCursor.Right,
n=(end - cursor_pos))
# This line actually applies the move to control's cursor
self._control.setTextCursor(cursor)
offset = end - start
# Move the local cursor object to the start of the match and
# complete.
cursor.movePosition(QtGui.QTextCursor.Left, n=offset)
self._complete_with_items(cursor, matches)
def _handle_execute_reply(self, msg):
""" Reimplemented to support prompt requests.
"""
msg_id = msg['parent_header'].get('msg_id')
info = self._request_info['execute'].get(msg_id)
if info and info.kind == 'prompt':
content = msg['content']
if content['status'] == 'aborted':
self._show_interpreter_prompt()
else:
number = content['execution_count'] + 1
self._show_interpreter_prompt(number)
self._request_info['execute'].pop(msg_id)
else:
super(IPythonWidget, self)._handle_execute_reply(msg)
def _handle_history_reply(self, msg):
""" Implemented to handle history tail replies, which are only supported
by the IPython kernel.
"""
content = msg['content']
if 'history' not in content:
self.log.error("History request failed: %r" % content)
if content.get('status', '') == 'aborted' and \
not self._retrying_history_request:
# a *different* action caused this request to be aborted, so
# we should try again.
self.log.error("Retrying aborted history request")
# prevent multiple retries of aborted requests:
self._retrying_history_request = True
# wait out the kernel's queue flush, which is currently timed at 0.1s
time.sleep(0.25)
self.kernel_client.history(hist_access_type='tail', n=1000)
else:
self._retrying_history_request = False
return
# reset retry flag
self._retrying_history_request = False
history_items = content['history']
self.log.debug("Received history reply with %i entries",
len(history_items))
items = []
last_cell = u""
for _, _, cell in history_items:
cell = cell.rstrip()
if cell != last_cell:
items.append(cell)
last_cell = cell
self._set_history(items)
def _insert_other_input(self, cursor, content):
"""Insert function for input from other frontends"""
cursor.beginEditBlock()
start = cursor.position()
n = content.get('execution_count', 0)
cursor.insertText('\n')
self._insert_html(cursor, self._make_in_prompt(n))
cursor.insertText(content['code'])
self._highlighter.rehighlightBlock(cursor.block())
cursor.endEditBlock()
def _handle_execute_input(self, msg):
"""Handle an execute_input message"""
self.log.debug("execute_input: %s", msg.get('content', ''))
if self.include_output(msg):
self._append_custom(self._insert_other_input,
msg['content'],
before_prompt=True)
def _handle_execute_result(self, msg):
"""Reimplemented for IPython-style "display hook"."""
if self.include_output(msg):
self.flush_clearoutput()
content = msg['content']
prompt_number = content.get('execution_count', 0)
data = content['data']
if 'text/plain' in data:
self._append_plain_text(self.output_sep, True)
self._append_html(self._make_out_prompt(prompt_number), True)
text = data['text/plain']
# If the repr is multiline, make sure we start on a new line,
# so that its lines are aligned.
if "\n" in text and not self.output_sep.endswith("\n"):
self._append_plain_text('\n', True)
self._append_plain_text(text + self.output_sep2, True)
def _handle_display_data(self, msg):
"""The base handler for the ``display_data`` message."""
# For now, we don't display data from other frontends, but we
# eventually will as this allows all frontends to monitor the display
# data. But we need to figure out how to handle this in the GUI.
if self.include_output(msg):
self.flush_clearoutput()
data = msg['content']['data']
metadata = msg['content']['metadata']
# In the regular IPythonWidget, we simply print the plain text
# representation.
if 'text/plain' in data:
text = data['text/plain']
self._append_plain_text(text, True)
# This newline seems to be needed for text and html output.
self._append_plain_text(u'\n', True)
def _handle_kernel_info_reply(self, rep):
"""Handle kernel info replies."""
content = rep['content']
if not self._guiref_loaded:
if content.get('implementation') == 'ipython':
self._load_guiref_magic()
self._guiref_loaded = True
self.kernel_banner = content.get('banner', '')
if self._starting:
# finish handling started channels
self._starting = False
super(IPythonWidget, self)._started_channels()
def _started_channels(self):
"""Reimplemented to make a history request and load %guiref."""
self._starting = True
# The reply will trigger %guiref load provided language=='python'
self.kernel_client.kernel_info()
self.kernel_client.history(hist_access_type='tail', n=1000)
def _load_guiref_magic(self):
"""Load %guiref magic."""
self.kernel_client.execute('\n'.join([
"try:",
" _usage",
"except:",
" from IPython.core import usage as _usage",
" get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')",
" del _usage",
]),
silent=True)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def execute_file(self, path, hidden=False):
""" Reimplemented to use the 'run' magic.
"""
# Use forward slashes on Windows to avoid escaping each separator.
if sys.platform == 'win32':
path = os.path.normpath(path).replace('\\', '/')
# Perhaps we should not be using %run directly, but while we
# are, it is necessary to quote or escape filenames containing spaces
# or quotes.
# In earlier code here, to minimize escaping, we sometimes quoted the
# filename with single quotes. But to do this, this code must be
# platform-aware, because run uses shlex rather than python string
# parsing, so that:
# * In Win: single quotes can be used in the filename without quoting,
# and we cannot use single quotes to quote the filename.
# * In *nix: we can escape double quotes in a double quoted filename,
# but can't escape single quotes in a single quoted filename.
# So to keep this code non-platform-specific and simple, we now only
# use double quotes to quote filenames, and escape when needed:
if ' ' in path or "'" in path or '"' in path:
path = '"%s"' % path.replace('"', '\\"')
self.execute('%%run %s' % path, hidden=hidden)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _process_execute_error(self, msg):
""" Reimplemented for IPython-style traceback formatting.
"""
content = msg['content']
traceback = '\n'.join(content['traceback']) + '\n'
if False:
# FIXME: For now, tracebacks come as plain text, so we can't use
# the html renderer yet. Once we refactor ultratb to produce
# properly styled tracebacks, this branch should be the default
traceback = traceback.replace(' ', ' ')
traceback = traceback.replace('\n', '<br/>')
ename = content['ename']
ename_styled = '<span class="error">%s</span>' % ename
traceback = traceback.replace(ename, ename_styled)
self._append_html(traceback)
else:
# This is the fallback for now, using plain text with ansi escapes
self._append_plain_text(traceback)
def _process_execute_payload(self, item):
""" Reimplemented to dispatch payloads to handler methods.
"""
handler = self._payload_handlers.get(item['source'])
if handler is None:
# We have no handler for this type of payload, simply ignore it
return False
else:
handler(item)
return True
def _show_interpreter_prompt(self, number=None):
""" Reimplemented for IPython-style prompts.
"""
# If a number was not specified, make a prompt number request.
if number is None:
msg_id = self.kernel_client.execute('', silent=True)
info = self._ExecutionRequest(msg_id, 'prompt')
self._request_info['execute'][msg_id] = info
return
# Show a new prompt and save information about it so that it can be
# updated later if the prompt number turns out to be wrong.
self._prompt_sep = self.input_sep
self._show_prompt(self._make_in_prompt(number), html=True)
block = self._control.document().lastBlock()
length = len(self._prompt)
self._previous_prompt_obj = self._PromptBlock(block, length, number)
# Update continuation prompt to reflect (possibly) new prompt length.
self._set_continuation_prompt(self._make_continuation_prompt(
self._prompt),
html=True)
def _show_interpreter_prompt_for_reply(self, msg):
""" Reimplemented for IPython-style prompts.
"""
# Update the old prompt number if necessary.
content = msg['content']
# abort replies do not have any keys:
if content['status'] == 'aborted':
if self._previous_prompt_obj:
previous_prompt_number = self._previous_prompt_obj.number
else:
previous_prompt_number = 0
else:
previous_prompt_number = content['execution_count']
if self._previous_prompt_obj and \
self._previous_prompt_obj.number != previous_prompt_number:
block = self._previous_prompt_obj.block
# Make sure the prompt block has not been erased.
if block.isValid() and block.text():
# Remove the old prompt and insert a new prompt.
cursor = QtGui.QTextCursor(block)
cursor.movePosition(QtGui.QTextCursor.Right,
QtGui.QTextCursor.KeepAnchor,
self._previous_prompt_obj.length)
prompt = self._make_in_prompt(previous_prompt_number)
self._prompt = self._insert_html_fetching_plain_text(
cursor, prompt)
# When the HTML is inserted, Qt blows away the syntax
# highlighting for the line, so we need to rehighlight it.
self._highlighter.rehighlightBlock(cursor.block())
self._previous_prompt_obj = None
# Show a new prompt with the kernel's estimated prompt number.
self._show_interpreter_prompt(previous_prompt_number + 1)
#---------------------------------------------------------------------------
# 'IPythonWidget' interface
#---------------------------------------------------------------------------
def set_default_style(self, colors='lightbg'):
""" Sets the widget style to the class defaults.
Parameters
----------
colors : str, optional (default lightbg)
Whether to use the default IPython light background or dark
background or B&W style.
"""
colors = colors.lower()
if colors == 'lightbg':
self.style_sheet = styles.default_light_style_sheet
self.syntax_style = styles.default_light_syntax_style
elif colors == 'linux':
self.style_sheet = styles.default_dark_style_sheet
self.syntax_style = styles.default_dark_syntax_style
elif colors == 'nocolor':
self.style_sheet = styles.default_bw_style_sheet
self.syntax_style = styles.default_bw_syntax_style
else:
raise KeyError("No such color scheme: %s" % colors)
#---------------------------------------------------------------------------
# 'IPythonWidget' protected interface
#---------------------------------------------------------------------------
def _edit(self, filename, line=None):
""" Opens a Python script for editing.
Parameters
----------
filename : str
A path to a local system file.
line : int, optional
A line of interest in the file.
"""
if self.custom_edit:
self.custom_edit_requested.emit(filename, line)
elif not self.editor:
self._append_plain_text(
'No default editor available.\n'
'Specify a GUI text editor in the `IPythonWidget.editor` '
'configurable to enable the %edit magic')
else:
try:
filename = '"%s"' % filename
if line and self.editor_line:
command = self.editor_line.format(filename=filename,
line=line)
else:
try:
command = self.editor.format()
except KeyError:
command = self.editor.format(filename=filename)
else:
command += ' ' + filename
except KeyError:
self._append_plain_text('Invalid editor command.\n')
else:
try:
Popen(command, shell=True)
except OSError:
msg = 'Opening editor with command "%s" failed.\n'
self._append_plain_text(msg % command)
def _make_in_prompt(self, number):
""" Given a prompt number, returns an HTML In prompt.
"""
try:
body = self.in_prompt % number
except TypeError:
# allow in_prompt to leave out number, e.g. '>>> '
from xml.sax.saxutils import escape
body = escape(self.in_prompt)
return '<span class="in-prompt">%s</span>' % body
def _make_continuation_prompt(self, prompt):
""" Given a plain text version of an In prompt, returns an HTML
continuation prompt.
"""
end_chars = '...: '
space_count = len(prompt.lstrip('\n')) - len(end_chars)
body = ' ' * space_count + end_chars
return '<span class="in-prompt">%s</span>' % body
def _make_out_prompt(self, number):
""" Given a prompt number, returns an HTML Out prompt.
"""
try:
body = self.out_prompt % number
except TypeError:
# allow out_prompt to leave out number, e.g. '<<< '
from xml.sax.saxutils import escape
body = escape(self.out_prompt)
return '<span class="out-prompt">%s</span>' % body
#------ Payload handlers --------------------------------------------------
# Payload handlers with a generic interface: each takes the opaque payload
# dict, unpacks it and calls the underlying functions with the necessary
# arguments.
def _handle_payload_edit(self, item):
self._edit(item['filename'], item['line_number'])
def _handle_payload_exit(self, item):
self._keep_kernel_on_exit = item['keepkernel']
self.exit_requested.emit(self)
def _handle_payload_next_input(self, item):
self.input_buffer = item['text']
def _handle_payload_page(self, item):
# Since the plain text widget supports only a very small subset of HTML
# and we have no control over the HTML source, we only page HTML
# payloads in the rich text widget.
data = item['data']
if 'text/html' in data and self.kind == 'rich':
self._page(data['text/html'], html=True)
else:
self._page(data['text/plain'], html=False)
#------ Trait change handlers --------------------------------------------
def _style_sheet_changed(self):
""" Set the style sheets of the underlying widgets.
"""
self.setStyleSheet(self.style_sheet)
if self._control is not None:
self._control.document().setDefaultStyleSheet(self.style_sheet)
bg_color = self._control.palette().window().color()
self._ansi_processor.set_background_color(bg_color)
if self._page_control is not None:
self._page_control.document().setDefaultStyleSheet(
self.style_sheet)
def _syntax_style_changed(self):
""" Set the style for the syntax highlighter.
"""
if self._highlighter is None:
# ignore premature calls
return
if self.syntax_style:
self._highlighter.set_style(self.syntax_style)
else:
self._highlighter.set_style_sheet(self.style_sheet)
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
return "Jupyter QtConsole {version}\n".format(version=__version__)
class LocalAuthenticator(Authenticator):
"""Base class for Authenticators that work with local Linux/UNIX users
Checks for local users, and can attempt to create them if they exist.
"""
create_system_users = Bool(False,
help="""
If set to True, will attempt to create local system users if they do not exist already.
Supports Linux and BSD variants only.
""").tag(config=True)
add_user_cmd = Command(help="""
The command to use for creating users as a list of strings
For each element in the list, the string USERNAME will be replaced with
the user's username. The username will also be appended as the final argument.
For Linux, the default value is:
['adduser', '-q', '--gecos', '""', '--disabled-password']
To specify a custom home directory, set this to:
['adduser', '-q', '--gecos', '""', '--home', '/customhome/USERNAME', '--disabled-password']
This will run the command:
adduser -q --gecos "" --home /customhome/river --disabled-password river
when the user 'river' is created.
""").tag(config=True)
@default('add_user_cmd')
def _add_user_cmd_default(self):
"""Guess the most likely-to-work adduser command for each platform"""
if sys.platform == 'darwin':
raise ValueError("I don't know how to create users on OS X")
elif which('pw'):
# Probably BSD
return ['pw', 'useradd', '-m']
else:
# This appears to be the Linux non-interactive adduser command:
return ['adduser', '-q', '--gecos', '""', '--disabled-password']
group_whitelist = Set(help="""
Whitelist all users from this UNIX group.
This makes the username whitelist ineffective.
""").tag(config=True)
@observe('group_whitelist')
def _group_whitelist_changed(self, change):
"""
Log a warning if both group_whitelist and user whitelist are set.
"""
if self.whitelist:
self.log.warning(
"Ignoring username whitelist because group whitelist supplied!"
)
def check_whitelist(self, username):
if self.group_whitelist:
return self.check_group_whitelist(username)
else:
return super().check_whitelist(username)
def check_group_whitelist(self, username):
"""
If group_whitelist is configured, check if authenticating user is part of group.
"""
if not self.group_whitelist:
return False
for grnam in self.group_whitelist:
try:
group = getgrnam(grnam)
except KeyError:
self.log.error('No such group: [%s]' % grnam)
continue
if username in group.gr_mem:
return True
return False
@gen.coroutine
def add_user(self, user):
"""Hook called whenever a new user is added
If self.create_system_users, the user will attempt to be created if it doesn't exist.
"""
user_exists = yield gen.maybe_future(self.system_user_exists(user))
if not user_exists:
if self.create_system_users:
yield gen.maybe_future(self.add_system_user(user))
else:
raise KeyError("User %s does not exist." % user.name)
yield gen.maybe_future(super().add_user(user))
@staticmethod
def system_user_exists(user):
"""Check if the user exists on the system"""
import pwd
try:
pwd.getpwnam(user.name)
except KeyError:
return False
else:
return True
def add_system_user(self, user):
"""Create a new local UNIX user on the system.
Tested to work on FreeBSD and Linux, at least.
"""
name = user.name
cmd = [arg.replace('USERNAME', name)
for arg in self.add_user_cmd] + [name]
self.log.info("Creating user: %s", ' '.join(map(pipes.quote, cmd)))
p = Popen(cmd, stdout=PIPE, stderr=STDOUT)
p.wait()
if p.returncode:
err = p.stdout.read().decode('utf8', 'replace')
raise RuntimeError("Failed to create system user %s: %s" %
(name, err))
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
""" A Qt frontend for a generic Python kernel.
"""
# The text to show when the kernel is (re)started.
banner = Unicode(config=True)
kernel_banner = Unicode()
# Whether to show the banner
_display_banner = Bool(False)
# An option and corresponding signal for overriding the default kernel
# interrupt behavior.
custom_interrupt = Bool(False)
custom_interrupt_requested = QtCore.Signal()
# An option and corresponding signals for overriding the default kernel
# restart behavior.
custom_restart = Bool(False)
custom_restart_kernel_died = QtCore.Signal(float)
custom_restart_requested = QtCore.Signal()
# Whether to automatically show calltips on open-parentheses.
enable_calltips = Bool(True, config=True,
help="Whether to draw information calltips on open-parentheses.")
clear_on_kernel_restart = Bool(True, config=True,
help="Whether to clear the console when the kernel is restarted")
confirm_restart = Bool(True, config=True,
help="Whether to ask for user confirmation when restarting kernel")
lexer_class = DottedObjectName(config=True,
help="The pygments lexer class to use."
)
def _lexer_class_changed(self, name, old, new):
lexer_class = import_item(new)
self.lexer = lexer_class()
def _lexer_class_default(self):
if py3compat.PY3:
return 'pygments.lexers.Python3Lexer'
else:
return 'pygments.lexers.PythonLexer'
lexer = Any()
def _lexer_default(self):
lexer_class = import_item(self.lexer_class)
return lexer_class()
# Emitted when a user visible 'execute_request' has been submitted to the
# kernel from the FrontendWidget. Contains the code to be executed.
executing = QtCore.Signal(object)
# Emitted when a user-visible 'execute_reply' has been received from the
# kernel and processed by the FrontendWidget. Contains the response message.
executed = QtCore.Signal(object)
# Emitted when an exit request has been received from the kernel.
exit_requested = QtCore.Signal(object)
_CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
_CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
_ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
_local_kernel = False
_highlighter = Instance(FrontendHighlighter, allow_none=True)
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(FrontendWidget, self).__init__(*args, **kw)
# FIXME: remove this when PySide min version is updated past 1.0.7
# forcefully disable calltips if PySide is < 1.0.7, because they crash
if qt.QT_API == qt.QT_API_PYSIDE:
import PySide
if PySide.__version_info__ < (1,0,7):
self.log.warn("PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__)
self.enable_calltips = False
# FrontendWidget protected variables.
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
self._hidden = False
self._highlighter = FrontendHighlighter(self, lexer=self.lexer)
self._kernel_manager = None
self._kernel_client = None
self._request_info = {}
self._request_info['execute'] = {}
self._callback_dict = {}
self._display_banner = True
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Configure actions.
action = self._copy_raw_action
key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
action.setEnabled(False)
action.setShortcut(QtGui.QKeySequence(key))
action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
action.triggered.connect(self.copy_raw)
self.copy_available.connect(action.setEnabled)
self.addAction(action)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Set flag for whether we are connected via localhost.
self._local_kernel = kw.get('local_kernel',
FrontendWidget._local_kernel)
# Whether or not a clear_output call is pending new output.
self._pending_clearoutput = False
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
def copy(self):
""" Copy the currently selected text to the clipboard, removing prompts.
"""
if self._page_control is not None and self._page_control.hasFocus():
self._page_control.copy()
elif self._control.hasFocus():
text = self._control.textCursor().selection().toPlainText()
if text:
# Remove prompts.
lines = text.splitlines()
lines = map(self._highlighter.transform_classic_prompt, lines)
lines = map(self._highlighter.transform_ipy_prompt, lines)
text = '\n'.join(lines)
# Needed to prevent errors when copying the prompt.
# See issue 264
try:
was_newline = text[-1] == '\n'
except IndexError:
was_newline = False
if was_newline: # user doesn't need newline
text = text[:-1]
QtGui.QApplication.clipboard().setText(text)
else:
self.log.debug("frontend widget : unknown copy target")
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
msg_id = self.kernel_client.execute(source, hidden)
self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'user')
self._hidden = hidden
if not hidden:
self.executing.emit(source)
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
# 3) There is no active selection.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
non_ws_before = bool(text[:self._get_input_buffer_cursor_column()].strip())
complete = non_ws_before and self._get_cursor().selectedText() == ''
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _context_menu_make(self, pos):
""" Reimplemented to add an action for raw copy.
"""
menu = super(FrontendWidget, self)._context_menu_make(pos)
for before_action in menu.actions():
if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
QtGui.QKeySequence.ExactMatch:
menu.insertAction(before_action, self._copy_raw_action)
break
return menu
def request_interrupt_kernel(self):
if self._executing:
self.interrupt_kernel()
def request_restart_kernel(self):
message = 'Are you sure you want to restart the kernel?'
self.restart_kernel(message, now=False)
def _event_filter_console_keypress(self, event):
""" Reimplemented for execution interruption and smart backspace.
"""
key = event.key()
if self._control_key_down(event.modifiers(), include_command=False):
if key == QtCore.Qt.Key_C and self._executing:
self.request_interrupt_kernel()
return True
elif key == QtCore.Qt.Key_Period:
self.request_restart_kernel()
return True
elif not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
if key == QtCore.Qt.Key_Backspace:
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(FrontendWidget, self)._event_filter_console_keypress(event)
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_clear_output(self, msg):
"""Handle clear output messages."""
if self.include_output(msg):
wait = msg['content'].get('wait', True)
if wait:
self._pending_clearoutput = True
else:
self.clear_output()
def _silent_exec_callback(self, expr, callback):
"""Silently execute `expr` in the kernel and call `callback` with reply
the `expr` is evaluated silently in the kernel (without) output in
the frontend. Call `callback` with the
`repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
Parameters
----------
expr : string
valid string to be executed by the kernel.
callback : function
function accepting one argument, as a string. The string will be
the `repr` of the result of evaluating `expr`
The `callback` is called with the `repr()` of the result of `expr` as
first argument. To get the object, do `eval()` on the passed value.
See Also
--------
_handle_exec_callback : private method, deal with calling callback with reply
"""
# generate uuid, which would be used as an indication of whether or
# not the unique request originated from here (can use msg id ?)
local_uuid = str(uuid.uuid1())
msg_id = self.kernel_client.execute('',
silent=True, user_expressions={ local_uuid:expr })
self._callback_dict[local_uuid] = callback
self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'silent_exec_callback')
def _handle_exec_callback(self, msg):
"""Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
Parameters
----------
msg : raw message send by the kernel containing an `user_expressions`
and having a 'silent_exec_callback' kind.
Notes
-----
This function will look for a `callback` associated with the
corresponding message id. Association has been made by
`_silent_exec_callback`. `callback` is then called with the `repr()`
of the value of corresponding `user_expressions` as argument.
`callback` is then removed from the known list so that any message
coming again with the same id won't trigger it.
"""
user_exp = msg['content'].get('user_expressions')
if not user_exp:
return
for expression in user_exp:
if expression in self._callback_dict:
self._callback_dict.pop(expression)(user_exp[expression])
def _handle_execute_reply(self, msg):
""" Handles replies for code execution.
"""
self.log.debug("execute_reply: %s", msg.get('content', ''))
msg_id = msg['parent_header']['msg_id']
info = self._request_info['execute'].get(msg_id)
# unset reading flag, because if execute finished, raw_input can't
# still be pending.
self._reading = False
# Note: If info is NoneType, this is ignored
if info and info.kind == 'user' and not self._hidden:
# Make sure that all output from the SUB channel has been processed
# before writing a new prompt.
self.kernel_client.iopub_channel.flush()
# Reset the ANSI style information to prevent bad text in stdout
# from messing up our colors. We're not a true terminal so we're
# allowed to do this.
if self.ansi_codes:
self._ansi_processor.reset_sgr()
content = msg['content']
status = content['status']
if status == 'ok':
self._process_execute_ok(msg)
elif status == 'aborted':
self._process_execute_abort(msg)
self._show_interpreter_prompt_for_reply(msg)
self.executed.emit(msg)
self._request_info['execute'].pop(msg_id)
elif info and info.kind == 'silent_exec_callback' and not self._hidden:
self._handle_exec_callback(msg)
self._request_info['execute'].pop(msg_id)
elif info and not self._hidden:
raise RuntimeError("Unknown handler for %s" % info.kind)
def _handle_error(self, msg):
""" Handle error messages.
"""
self._process_execute_error(msg)
def _handle_input_request(self, msg):
""" Handle requests for raw_input.
"""
self.log.debug("input: %s", msg.get('content', ''))
if self._hidden:
raise RuntimeError('Request for raw input during hidden execution.')
# Make sure that all output from the SUB channel has been processed
# before entering readline mode.
self.kernel_client.iopub_channel.flush()
def callback(line):
self.kernel_client.input(line)
if self._reading:
self.log.debug("Got second input request, assuming first was interrupted.")
self._reading = False
self._readline(msg['content']['prompt'], callback=callback, password=msg['content']['password'])
def _kernel_restarted_message(self, died=True):
msg = "Kernel died, restarting" if died else "Kernel restarting"
self._append_html("<br>%s<hr><br>" % msg,
before_prompt=False
)
def _handle_kernel_died(self, since_last_heartbeat):
"""Handle the kernel's death (if we do not own the kernel).
"""
self.log.warn("kernel died: %s", since_last_heartbeat)
if self.custom_restart:
self.custom_restart_kernel_died.emit(since_last_heartbeat)
else:
self._kernel_restarted_message(died=True)
self.reset()
def _handle_kernel_restarted(self, died=True):
"""Notice that the autorestarter restarted the kernel.
There's nothing to do but show a message.
"""
self.log.warn("kernel restarted")
self._kernel_restarted_message(died=died)
self.reset()
def _handle_inspect_reply(self, rep):
"""Handle replies for call tips."""
self.log.debug("oinfo: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('call_tip')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
content = rep['content']
if content.get('status') == 'ok' and content.get('found', False):
self._call_tip_widget.show_inspect_data(content)
def _handle_execute_result(self, msg):
""" Handle display hook output.
"""
self.log.debug("execute_result: %s", msg.get('content', ''))
if self.include_output(msg):
self.flush_clearoutput()
text = msg['content']['data']
self._append_plain_text(text + '\n', before_prompt=True)
def _handle_stream(self, msg):
""" Handle stdout, stderr, and stdin.
"""
self.log.debug("stream: %s", msg.get('content', ''))
if self.include_output(msg):
self.flush_clearoutput()
self.append_stream(msg['content']['text'])
def _handle_shutdown_reply(self, msg):
""" Handle shutdown signal, only if from other console.
"""
self.log.debug("shutdown: %s", msg.get('content', ''))
restart = msg.get('content', {}).get('restart', False)
if not self._hidden and not self.from_here(msg):
# got shutdown reply, request came from session other than ours
if restart:
# someone restarted the kernel, handle it
self._handle_kernel_restarted(died=False)
else:
# kernel was shutdown permanently
# this triggers exit_requested if the kernel was local,
# and a dialog if the kernel was remote,
# so we don't suddenly clear the qtconsole without asking.
if self._local_kernel:
self.exit_requested.emit(self)
else:
title = self.window().windowTitle()
reply = QtGui.QMessageBox.question(self, title,
"Kernel has been shutdown permanently. "
"Close the Console?",
QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.exit_requested.emit(self)
def _handle_status(self, msg):
"""Handle status message"""
# This is where a busy/idle indicator would be triggered,
# when we make one.
state = msg['content'].get('execution_state', '')
if state == 'starting':
# kernel started while we were running
if self._executing:
self._handle_kernel_restarted(died=True)
elif state == 'idle':
pass
elif state == 'busy':
pass
def _started_channels(self):
""" Called when the KernelManager channels have started listening or
when the frontend is assigned an already listening KernelManager.
"""
self.reset(clear=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def copy_raw(self):
""" Copy the currently selected text to the clipboard without attempting
to remove prompts or otherwise alter the text.
"""
self._control.copy()
def interrupt_kernel(self):
""" Attempts to interrupt the running kernel.
Also unsets _reading flag, to avoid runtime errors
if raw_input is called again.
"""
if self.custom_interrupt:
self._reading = False
self.custom_interrupt_requested.emit()
elif self.kernel_manager:
self._reading = False
self.kernel_manager.interrupt_kernel()
else:
self._append_plain_text('Cannot interrupt a kernel I did not start.\n')
def reset(self, clear=False):
""" Resets the widget to its initial state if ``clear`` parameter
is True, otherwise
prints a visual indication of the fact that the kernel restarted, but
does not clear the traces from previous usage of the kernel before it
was restarted. With ``clear=True``, it is similar to ``%clear``, but
also re-writes the banner and aborts execution if necessary.
"""
if self._executing:
self._executing = False
self._request_info['execute'] = {}
self._reading = False
self._highlighter.highlighting_on = False
if clear:
self._control.clear()
if self._display_banner:
self._append_plain_text(self.banner)
if self.kernel_banner:
self._append_plain_text(self.kernel_banner)
# update output marker for stdout/stderr, so that startup
# messages appear after banner:
self._show_interpreter_prompt()
def restart_kernel(self, message, now=False):
""" Attempts to restart the running kernel.
"""
# FIXME: now should be configurable via a checkbox in the dialog. Right
# now at least the heartbeat path sets it to True and the manual restart
# to False. But those should just be the pre-selected states of a
# checkbox that the user could override if so desired. But I don't know
# enough Qt to go implementing the checkbox now.
if self.custom_restart:
self.custom_restart_requested.emit()
return
if self.kernel_manager:
# Pause the heart beat channel to prevent further warnings.
self.kernel_client.hb_channel.pause()
# Prompt the user to restart the kernel. Un-pause the heartbeat if
# they decline. (If they accept, the heartbeat will be un-paused
# automatically when the kernel is restarted.)
if self.confirm_restart:
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
result = QtGui.QMessageBox.question(self, 'Restart kernel?',
message, buttons)
do_restart = result == QtGui.QMessageBox.Yes
else:
# confirm_restart is False, so we don't need to ask user
# anything, just do the restart
do_restart = True
if do_restart:
try:
self.kernel_manager.restart_kernel(now=now)
except RuntimeError as e:
self._append_plain_text(
'Error restarting kernel: %s\n' % e,
before_prompt=True
)
else:
self._append_html("<br>Restarting kernel...\n<hr><br>",
before_prompt=True,
)
else:
self.kernel_client.hb_channel.unpause()
else:
self._append_plain_text(
'Cannot restart a Kernel I did not start\n',
before_prompt=True
)
def append_stream(self, text):
"""Appends text to the output stream."""
# Most consoles treat tabs as being 8 space characters. Convert tabs
# to spaces so that output looks as expected regardless of this
# widget's tab width.
text = text.expandtabs(8)
self._append_plain_text(text, before_prompt=True)
self._control.moveCursor(QtGui.QTextCursor.End)
def flush_clearoutput(self):
"""If a clearoutput is pending, execute it."""
if self._pending_clearoutput:
self._pending_clearoutput = False
self.clear_output()
def clear_output(self):
"""Clears the current line of output."""
cursor = self._control.textCursor()
cursor.beginEditBlock()
cursor.movePosition(cursor.StartOfLine, cursor.KeepAnchor)
cursor.insertText('')
cursor.endEditBlock()
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _auto_call_tip(self):
"""Trigger call tip automatically on open parenthesis
Call tips can be requested explcitly with `_call_tip`.
"""
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) == '(':
# trigger auto call tip on open paren
self._call_tip()
def _call_tip(self):
"""Shows a call tip, if appropriate, at the current cursor location."""
# Decide if it makes sense to show a call tip
if not self.enable_calltips or not self.kernel_client.shell_channel.is_alive():
return False
cursor_pos = self._get_input_buffer_cursor_pos()
code = self.input_buffer
# Send the metadata request to the kernel
msg_id = self.kernel_client.inspect(code, cursor_pos)
pos = self._get_cursor().position()
self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
return True
def _complete(self):
""" Performs completion at the current cursor location.
"""
# Send the completion request to the kernel
msg_id = self.kernel_client.complete(
code=self.input_buffer,
cursor_pos=self._get_input_buffer_cursor_pos(),
)
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _process_execute_abort(self, msg):
""" Process a reply for an aborted execution request.
"""
self._append_plain_text("ERROR: execution aborted\n")
def _process_execute_error(self, msg):
""" Process a reply for an execution request that resulted in an error.
"""
content = msg['content']
# If a SystemExit is passed along, this means exit() was called - also
# all the ipython %exit magic syntax of '-k' to be used to keep
# the kernel running
if content['ename']=='SystemExit':
keepkernel = content['evalue']=='-k' or content['evalue']=='True'
self._keep_kernel_on_exit = keepkernel
self.exit_requested.emit(self)
else:
traceback = ''.join(content['traceback'])
self._append_plain_text(traceback)
def _process_execute_ok(self, msg):
""" Process a reply for a successful execution request.
"""
payload = msg['content'].get('payload', [])
for item in payload:
if not self._process_execute_payload(item):
warning = 'Warning: received unknown payload of type %s'
print(warning % repr(item['source']))
def _process_execute_payload(self, item):
""" Process a single payload item from the list of payload items in an
execution reply. Returns whether the payload was handled.
"""
# The basic FrontendWidget doesn't handle payloads, as they are a
# mechanism for going beyond the standard Python interpreter model.
return False
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self._show_prompt('>>> ')
def _show_interpreter_prompt_for_reply(self, msg):
""" Shows a prompt for the interpreter given an 'execute_reply' message.
"""
self._show_interpreter_prompt()
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._auto_call_tip()
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
""" Returns the standard Python banner.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
class PromptManager(Configurable):
"""This is the primary interface for producing IPython's prompts."""
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
color_scheme_table = Instance(coloransi.ColorSchemeTable, allow_none=True)
color_scheme = Unicode('Linux', config=True)
def _color_scheme_changed(self, name, new_value):
self.color_scheme_table.set_active_scheme(new_value)
for pname in ['in', 'in2', 'out', 'rewrite']:
# We need to recalculate the number of invisible characters
self.update_prompt(pname)
lazy_evaluate_fields = Dict(help="""
This maps field names used in the prompt templates to functions which
will be called when the prompt is rendered. This allows us to include
things like the current time in the prompts. Functions are only called
if they are used in the prompt.
""")
def _lazy_evaluate_fields_default(self):
return lazily_evaluate.copy()
in_template = Unicode(
'In [\\#]: ',
config=True,
help="Input prompt. '\\#' will be transformed to the prompt number")
in2_template = Unicode(' .\\D.: ',
config=True,
help="Continuation prompt.")
out_template = Unicode(
'Out[\\#]: ',
config=True,
help="Output prompt. '\\#' will be transformed to the prompt number")
justify = Bool(True,
config=True,
help="""
If True (default), each prompt will be right-aligned with the
preceding one.
""")
# We actually store the expanded templates here:
templates = Dict()
# The number of characters in the last prompt rendered, not including
# colour characters.
width = Int()
txtwidth = Int() # Not including right-justification
# The number of characters in each prompt which don't contribute to width
invisible_chars = Dict()
def _invisible_chars_default(self):
return {'in': 0, 'in2': 0, 'out': 0, 'rewrite': 0}
def __init__(self, shell, **kwargs):
super(PromptManager, self).__init__(shell=shell, **kwargs)
# Prepare colour scheme table
self.color_scheme_table = coloransi.ColorSchemeTable(
[PColNoColors, PColLinux, PColLightBG], self.color_scheme)
self._formatter = UserNSFormatter(shell)
# Prepare templates & numbers of invisible characters
self.update_prompt('in', self.in_template)
self.update_prompt('in2', self.in2_template)
self.update_prompt('out', self.out_template)
self.update_prompt('rewrite')
self.on_trait_change(self._update_prompt_trait,
['in_template', 'in2_template', 'out_template'])
def update_prompt(self, name, new_template=None):
"""This is called when a prompt template is updated. It processes
abbreviations used in the prompt template (like \#) and calculates how
many invisible characters (ANSI colour escapes) the resulting prompt
contains.
It is also called for each prompt on changing the colour scheme. In both
cases, traitlets should take care of calling this automatically.
"""
if new_template is not None:
self.templates[name] = multiple_replace(prompt_abbreviations,
new_template)
# We count invisible characters (colour escapes) on the last line of the
# prompt, to calculate the width for lining up subsequent prompts.
invis_chars = _invisible_characters(self._render(name, color=True))
self.invisible_chars[name] = invis_chars
def _update_prompt_trait(self, traitname, new_template):
name = traitname[:-9] # Cut off '_template'
self.update_prompt(name, new_template)
def _render(self, name, color=True, **kwargs):
"""Render but don't justify, or update the width or txtwidth attributes.
"""
if name == 'rewrite':
return self._render_rewrite(color=color)
if color:
scheme = self.color_scheme_table.active_colors
if name == 'out':
colors = color_lists['normal']
colors.number, colors.prompt, colors.normal = \
scheme.out_number, scheme.out_prompt, scheme.normal
else:
colors = color_lists['inp']
colors.number, colors.prompt, colors.normal = \
scheme.in_number, scheme.in_prompt, scheme.in_normal
if name == 'in2':
colors.prompt = scheme.in_prompt2
else:
# No color
colors = color_lists['nocolor']
colors.number, colors.prompt, colors.normal = '', '', ''
count = self.shell.execution_count # Shorthand
# Build the dictionary to be passed to string formatting
fmtargs = dict(color=colors,
count=count,
dots="." * len(str(count)),
width=self.width,
txtwidth=self.txtwidth)
fmtargs.update(self.lazy_evaluate_fields)
fmtargs.update(kwargs)
# Prepare the prompt
prompt = colors.prompt + self.templates[name] + colors.normal
# Fill in required fields
return self._formatter.format(prompt, **fmtargs)
def _render_rewrite(self, color=True):
"""Render the ---> rewrite prompt."""
if color:
scheme = self.color_scheme_table.active_colors
# We need a non-input version of these escapes
color_prompt = scheme.in_prompt.replace("\001",
"").replace("\002", "")
color_normal = scheme.normal
else:
color_prompt, color_normal = '', ''
return color_prompt + "-> ".rjust(self.txtwidth, "-") + color_normal
def render(self, name, color=True, just=None, **kwargs):
"""
Render the selected prompt.
Parameters
----------
name : str
Which prompt to render. One of 'in', 'in2', 'out', 'rewrite'
color : bool
If True (default), include ANSI escape sequences for a coloured prompt.
just : bool
If True, justify the prompt to the width of the last prompt. The
default is stored in self.justify.
**kwargs :
Additional arguments will be passed to the string formatting operation,
so they can override the values that would otherwise fill in the
template.
Returns
-------
A string containing the rendered prompt.
"""
res = self._render(name, color=color, **kwargs)
# Handle justification of prompt
invis_chars = self.invisible_chars[name] if color else 0
self.txtwidth = _lenlastline(res) - invis_chars
just = self.justify if (just is None) else just
# If the prompt spans more than one line, don't try to justify it:
if just and name != 'in' and ('\n' not in res) and ('\r' not in res):
res = res.rjust(self.width + invis_chars)
self.width = _lenlastline(res) - invis_chars
return res
class Repo2Docker(Application):
"""An application for converting git repositories to docker images"""
name = "jupyter-repo2docker"
version = __version__
description = __doc__
@default("log_level")
def _default_log_level(self):
"""The application's default log level"""
return logging.INFO
git_workdir = Unicode(
None,
config=True,
allow_none=True,
help="""
Working directory to use for check out of git repositories.
The default is to use the system's temporary directory. Should be
somewhere ephemeral, such as /tmp.
""",
)
subdir = Unicode(
"",
config=True,
help="""
Subdirectory of the git repository to examine.
Defaults to ''.
""",
)
cache_from = List(
[],
config=True,
help="""
List of images to try & re-use cached image layers from.
Docker only tries to re-use image layers from images built locally,
not pulled from a registry. We can ask it to explicitly re-use layers
from non-locally built images by through the 'cache_from' parameter.
""",
)
buildpacks = List(
[
LegacyBinderDockerBuildPack,
DockerBuildPack,
JuliaProjectTomlBuildPack,
JuliaRequireBuildPack,
NixBuildPack,
RBuildPack,
CondaBuildPack,
PipfileBuildPack,
PythonBuildPack,
],
config=True,
help="""
Ordered list of BuildPacks to try when building a git repository.
""",
)
extra_build_kwargs = Dict(
{},
help="""
extra kwargs to limit CPU quota when building a docker image.
Dictionary that allows the user to set the desired runtime flag
to configure the amount of access to CPU resources your container has.
Reference https://docs.docker.com/config/containers/resource_constraints/#cpu
""",
config=True,
)
extra_run_kwargs = Dict(
{},
help="""
extra kwargs to limit CPU quota when running a docker image.
Dictionary that allows the user to set the desired runtime flag
to configure the amount of access to CPU resources your container has.
Reference https://docs.docker.com/config/containers/resource_constraints/#cpu
""",
config=True,
)
extra_build_args = Dict(
{},
help="""
Regular Docker build-time arguments that will be passed as
--build-arg key=value.
Reference https://docs.docker.com/engine/reference/commandline/build/
""",
config=True,
)
default_buildpack = Any(
PythonBuildPack,
config=True,
help="""
The default build pack to use when no other buildpacks are found.
""",
)
# Git is our content provider of last resort. This is to maintain the
# old behaviour when git and local directories were the only supported
# content providers. We can detect local directories from the path, but
# detecting if something will successfully `git clone` is very hard if all
# you can do is look at the path/URL to it.
content_providers = List(
[
contentproviders.Local,
contentproviders.Zenodo,
contentproviders.Figshare,
contentproviders.Dataverse,
contentproviders.Hydroshare,
contentproviders.Mercurial,
contentproviders.Git,
],
config=True,
help="""
Ordered list by priority of ContentProviders to try in turn to fetch
the contents specified by the user.
""",
)
build_memory_limit = ByteSpecification(
0,
help="""
Total memory that can be used by the docker image building process.
Set to 0 for no limits.
""",
config=True,
)
volumes = Dict(
{},
help="""
Volumes to mount when running the container.
Only used when running, not during build process!
Use a key-value pair, with the key being the volume source &
value being the destination volume.
Both source and destination can be relative. Source is resolved
relative to the current working directory on the host, and
destination is resolved relative to the working directory of the
image - ($HOME by default)
""",
config=True,
)
user_id = Int(
help="""
UID of the user to create inside the built image.
Should be a uid that is not currently used by anything in the image.
Defaults to uid of currently running user, since that is the most
common case when running r2d manually.
Might not affect Dockerfile builds.
""",
config=True,
)
@default("user_id")
def _user_id_default(self):
"""
Default user_id to current running user.
"""
return os.geteuid()
user_name = Unicode(
"jovyan",
help="""
Username of the user to create inside the built image.
Should be a username that is not currently used by anything in the
image, and should conform to the restrictions on user names for Linux.
Defaults to username of currently running user, since that is the most
common case when running repo2docker manually.
""",
config=True,
)
@default("user_name")
def _user_name_default(self):
"""
Default user_name to current running user.
"""
return getpass.getuser()
appendix = Unicode(
config=True,
help="""
Appendix of Dockerfile commands to run at the end of the build.
Can be used to customize the resulting image after all
standard build steps finish.
""",
)
json_logs = Bool(
False,
help="""
Log output in structured JSON format.
Useful when stdout is consumed by other tools
""",
config=True,
)
repo = Unicode(
".",
help="""
Specification of repository to build image for.
Could be local path or git URL.
""",
config=True,
)
ref = Unicode(
None,
help="""
Git ref that should be built.
If repo is a git repository, this ref is checked out
in a local clone before repository is built.
""",
config=True,
allow_none=True,
)
cleanup_checkout = Bool(
False,
help="""
Delete source repository after building is done.
Useful when repo2docker is doing the git cloning
""",
config=True,
)
output_image_spec = Unicode(
"",
help="""
Docker Image name:tag to tag the built image with.
Required parameter.
""",
config=True,
)
push = Bool(
False,
help="""
Set to true to push docker image after building
""",
config=True,
)
run = Bool(
False,
help="""
Run docker image after building
""",
config=True,
)
# FIXME: Refactor class to be able to do --no-build without needing
# deep support for it inside other code
dry_run = Bool(
False,
help="""
Do not actually build the docker image, just simulate it.
""",
config=True,
)
# FIXME: Refactor classes to separate build & run steps
run_cmd = List(
[],
help="""
Command to run when running the container
When left empty, a jupyter notebook is run.
""",
config=True,
)
all_ports = Bool(
False,
help="""
Publish all declared ports from container while running.
Equivalent to -P option to docker run
""",
config=True,
)
ports = Dict(
{},
help="""
Port mappings to establish when running the container.
Equivalent to -p {key}:{value} options to docker run.
{key} refers to port inside container, and {value}
refers to port / host:port in the host
""",
config=True,
)
environment = List(
[],
help="""
Environment variables to set when running the built image.
Each item must be a string formatted as KEY=VALUE
""",
config=True,
)
target_repo_dir = Unicode(
"",
help="""
Path inside the image where contents of the repositories are copied to,
and where all the build operations (such as postBuild) happen.
Defaults to ${HOME} if not set
""",
config=True,
)
template = Unicode(
"",
help="""
Jinja template used to render the Dockerfile.
""",
config=True,
)
entrypoint_file = Unicode(
"",
help="""
Path to a file that will be used as an entry point in the Docker image.
""",
config=True,
)
def fetch(self, url, ref, checkout_path):
"""Fetch the contents of `url` and place it in `checkout_path`.
The `ref` parameter specifies what "version" of the contents should be
fetched. In the case of a git repository `ref` is the SHA-1 of a commit.
Iterate through possible content providers until a valid provider,
based on URL, is found.
"""
picked_content_provider = None
for ContentProvider in self.content_providers:
cp = ContentProvider()
spec = cp.detect(url, ref=ref)
if spec is not None:
picked_content_provider = cp
self.log.info(
"Picked {cp} content "
"provider.\n".format(cp=cp.__class__.__name__)
)
break
if picked_content_provider is None:
self.log.error(
"No matching content provider found for " "{url}.".format(url=url)
)
for log_line in picked_content_provider.fetch(
spec, checkout_path, yield_output=self.json_logs
):
self.log.info(log_line, extra=dict(phase="fetching"))
if not self.output_image_spec:
self.output_image_spec = (
"r2d" + escapism.escape(self.repo, escape_char="-").lower()
)
# if we are building from a subdirectory include that in the
# image name so we can tell builds from different sub-directories
# apart.
if self.subdir:
self.output_image_spec += escapism.escape(
self.subdir, escape_char="-"
).lower()
if picked_content_provider.content_id is not None:
self.output_image_spec += picked_content_provider.content_id
else:
self.output_image_spec += str(int(time.time()))
def json_excepthook(self, etype, evalue, traceback):
"""Called on an uncaught exception when using json logging
Avoids non-JSON output on errors when using --json-logs
"""
self.log.error(
"Error during build: %s",
evalue,
exc_info=(etype, evalue, traceback),
extra=dict(phase="failed"),
)
def initialize(self):
"""Init repo2docker configuration before start"""
# FIXME: Remove this function, move it to setters / traitlet reactors
if self.json_logs:
# register JSON excepthook to avoid non-JSON output on errors
sys.excepthook = self.json_excepthook
# Need to reset existing handlers, or we repeat messages
logHandler = logging.StreamHandler()
formatter = jsonlogger.JsonFormatter()
logHandler.setFormatter(formatter)
self.log = logging.getLogger("repo2docker")
self.log.handlers = []
self.log.addHandler(logHandler)
self.log.setLevel(self.log_level)
else:
# due to json logger stuff above,
# our log messages include carriage returns, newlines, etc.
# remove the additional newline from the stream handler
self.log.handlers[0].terminator = ""
# We don't want a [Repo2Docker] on all messages
self.log.handlers[0].formatter = logging.Formatter(fmt="%(message)s")
if self.dry_run and (self.run or self.push):
raise ValueError("Cannot push or run image if we are not building it")
if self.volumes and not self.run:
raise ValueError("Cannot mount volumes if container is not run")
def push_image(self):
"""Push docker image to registry"""
client = docker.APIClient(version="auto", **kwargs_from_env())
# Build a progress setup for each layer, and only emit per-layer
# info every 1.5s
progress_layers = {}
layers = {}
last_emit_time = time.time()
for chunk in client.push(self.output_image_spec, stream=True):
# each chunk can be one or more lines of json events
# split lines here in case multiple are delivered at once
for line in chunk.splitlines():
line = line.decode("utf-8", errors="replace")
try:
progress = json.loads(line)
except Exception as e:
self.log.warning("Not a JSON progress line: %r", line)
continue
if "error" in progress:
self.log.error(progress["error"], extra=dict(phase="failed"))
raise docker.errors.ImageLoadError(progress["error"])
if "id" not in progress:
continue
# deprecated truncated-progress data
if "progressDetail" in progress and progress["progressDetail"]:
progress_layers[progress["id"]] = progress["progressDetail"]
else:
progress_layers[progress["id"]] = progress["status"]
# include full progress data for each layer in 'layers' data
layers[progress["id"]] = progress
if time.time() - last_emit_time > 1.5:
self.log.info(
"Pushing image\n",
extra=dict(
progress=progress_layers, layers=layers, phase="pushing"
),
)
last_emit_time = time.time()
self.log.info(
"Successfully pushed {}".format(self.output_image_spec),
extra=dict(phase="pushing"),
)
def run_image(self):
"""Run docker container from built image
and wait for it to finish.
"""
container = self.start_container()
self.wait_for_container(container)
def start_container(self):
"""Start docker container from built image
Returns running container
"""
client = docker.from_env(version="auto")
docker_host = os.environ.get("DOCKER_HOST")
if docker_host:
host_name = urlparse(docker_host).hostname
else:
host_name = "127.0.0.1"
self.hostname = host_name
if not self.run_cmd:
port = str(self._get_free_port())
self.port = port
# To use the option --NotebookApp.custom_display_url
# make sure the base-notebook image is updated:
# docker pull jupyter/base-notebook
run_cmd = [
"jupyter",
"notebook",
"--ip",
"0.0.0.0",
"--port",
port,
"--NotebookApp.custom_display_url=http://{}:{}".format(host_name, port),
]
ports = {"%s/tcp" % port: port}
else:
# run_cmd given by user, if port is also given then pass it on
run_cmd = self.run_cmd
if self.ports:
ports = self.ports
else:
ports = {}
# store ports on self so they can be retrieved in tests
self.ports = ports
container_volumes = {}
if self.volumes:
api_client = docker.APIClient(
version="auto", **docker.utils.kwargs_from_env()
)
image = api_client.inspect_image(self.output_image_spec)
# Buildkit uses Config/WorkingDir not ContainerConfig/WorkingDir
image_workdir = image["Config"]["WorkingDir"]
for k, v in self.volumes.items():
container_volumes[os.path.abspath(k)] = {
"bind": v if v.startswith("/") else os.path.join(image_workdir, v),
"mode": "rw",
}
run_kwargs = dict(
publish_all_ports=self.all_ports,
ports=ports,
detach=True,
command=run_cmd,
volumes=container_volumes,
environment=self.environment,
)
run_kwargs.update(self.extra_run_kwargs)
container = client.containers.run(self.output_image_spec, **run_kwargs)
while container.status == "created":
time.sleep(0.5)
container.reload()
return container
def wait_for_container(self, container):
"""Wait for a container to finish
Displaying logs while it's running
"""
try:
for line in container.logs(stream=True):
self.log.info(line.decode("utf-8"), extra=dict(phase="running"))
finally:
container.reload()
if container.status == "running":
self.log.info("Stopping container...\n", extra=dict(phase="running"))
container.kill()
exit_code = container.attrs["State"]["ExitCode"]
container.remove()
if exit_code:
sys.exit(exit_code)
def _get_free_port(self):
"""
Hacky method to get a free random port on local host
"""
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
return port
def find_image(self):
# if this is a dry run it is Ok for dockerd to be unreachable so we
# always return False for dry runs.
if self.dry_run:
return False
# check if we already have an image for this content
client = docker.APIClient(version="auto", **kwargs_from_env())
for image in client.images():
if image["RepoTags"] is not None:
for tag in image["RepoTags"]:
if tag == self.output_image_spec + ":latest":
return True
return False
def build(self):
"""
Build docker image
"""
# Check if r2d can connect to docker daemon
if not self.dry_run:
try:
docker_client = DockerCLI()
except DockerException as e:
self.log.error(
"\nDocker client initialization error: %s.\nCheck if docker is running on the host.\n",
e,
)
self.exit(1)
# If the source to be executed is a directory, continue using the
# directory. In the case of a local directory, it is used as both the
# source and target. Reusing a local directory seems better than
# making a copy of it as it might contain large files that would be
# expensive to copy.
if os.path.isdir(self.repo):
checkout_path = self.repo
else:
if self.git_workdir is None:
checkout_path = tempfile.mkdtemp(prefix="repo2docker")
else:
checkout_path = self.git_workdir
try:
self.fetch(self.repo, self.ref, checkout_path)
if self.find_image():
self.log.info(
"Reusing existing image ({}), not "
"building.".format(self.output_image_spec)
)
# no need to build, so skip to the end by `return`ing here
# this will still execute the finally clause and let's us
# avoid having to indent the build code by an extra level
return
if self.subdir:
checkout_path = os.path.join(checkout_path, self.subdir)
if not os.path.isdir(checkout_path):
self.log.error(
"Subdirectory %s does not exist",
self.subdir,
extra=dict(phase="failure"),
)
raise FileNotFoundError("Could not find {}".format(checkout_path))
with chdir(checkout_path):
for BP in self.buildpacks:
bp = BP()
if bp.detect():
picked_buildpack = bp
break
else:
picked_buildpack = self.default_buildpack()
picked_buildpack.appendix = self.appendix
if self.template:
picked_buildpack.template = self.template
if self.entrypoint_file:
picked_buildpack.entrypoint_file = self.entrypoint_file
# Add metadata labels
picked_buildpack.labels["repo2docker.version"] = self.version
repo_label = "local" if os.path.isdir(self.repo) else self.repo
picked_buildpack.labels["repo2docker.repo"] = repo_label
picked_buildpack.labels["repo2docker.ref"] = self.ref
if self.dry_run:
print(picked_buildpack.render())
else:
self.log.debug(
picked_buildpack.render(), extra=dict(phase="building")
)
if self.user_id == 0:
raise ValueError(
"Root as the primary user in the image is not permitted."
)
build_args = {
"NB_USER": self.user_name,
"NB_UID": str(self.user_id),
}
if self.target_repo_dir:
build_args["REPO_DIR"] = self.target_repo_dir
self.log.info(
"Using %s builder\n",
bp.__class__.__name__,
extra=dict(phase="building"),
)
build_args.update(self.extra_build_args)
for l in picked_buildpack.build(
docker_client,
self.output_image_spec,
self.build_memory_limit,
build_args,
self.cache_from,
self.extra_build_kwargs,
):
if "stream" in l:
self.log.info(l["stream"], extra=dict(phase="building"))
elif "error" in l:
self.log.info(l["error"], extra=dict(phase="failure"))
raise docker.errors.BuildError(l["error"], build_log="")
elif "status" in l:
self.log.info(
"Fetching base image...\r", extra=dict(phase="building")
)
else:
self.log.info(json.dumps(l), extra=dict(phase="building"))
finally:
# Cleanup checkout if necessary
if self.cleanup_checkout:
shutil.rmtree(checkout_path, ignore_errors=True)
def start(self):
self.build()
if self.push:
self.push_image()
if self.run:
self.run_image()
class Turtle(widgets.DOMWidget):
""""""
_view_name = Unicode('TurtleView').tag(sync=True)
_model_name = Unicode('TurtleModel').tag(sync=True)
_view_module = Unicode('ipython-turtle-widget').tag(sync=True)
_model_module = Unicode('ipython-turtle-widget').tag(sync=True)
_canvas_fixed = Bool(True).tag(sync=True)
_canvas_width = Int(320).tag(sync=True)
_canvas_height = Int(320).tag(sync=True)
_turtle_on = Bool(True).tag(sync=True)
_pen_on = True
_turtle_height = Int(20).tag(sync=True)
_turtle_width = Int(10).tag(sync=True)
_turtle_location_x = Float(0.0).tag(sync=True)
_turtle_location_y = Float(0.0).tag(sync=True)
_turtle_heading = Int(90).tag(sync=True)
_turtle_heading_x = Float(0).tag(sync=True)
_turtle_heading_y = Float(1).tag(sync=True)
_line = Unicode('').tag(sync=True)
def __init__(self, width=320, height=320, fixed=True):
widgets.DOMWidget.__init__(self)
self._canvas_width = width
self._canvas_height = height
self._canvas_fixed = fixed
self._reset()
def _reset(self):
self._turtle_on = True
self._pen_on = True
self._turtle_location_x = 0
self._turtle_location_y = 0
self._turtle_heading = 90
self._turtle_heading_x = 0.0
self._turtle_heading_y = 1.0
def position(self):
return (self._turtle_location_x, self._turtle_location_y)
def forward(self, length):
start = "{} {}".format(self._turtle_location_x,
self._turtle_location_y)
self._turtle_location_x += length * self._turtle_heading_x
self._turtle_location_y += length * self._turtle_heading_y
end = " {} {}".format(self._turtle_location_x, self._turtle_location_y)
if self._pen_on:
self._line = start + end
def back(self, length):
self.forward(-length)
def heading(self):
return self._turtle_heading
def left(self, degree):
self._turtle_heading += degree
hx = math.cos(math.radians(self._turtle_heading))
hy = math.sin(math.radians(self._turtle_heading))
self._turtle_heading_x = hx
self._turtle_heading_y = hy
def right(self, degree):
self.left(-degree)
def penup(self):
self._pen_on = False
def pendown(self):
self._pen_on = True
def isdown(self):
return self._pen_on
def hideturtle(self):
self._turtle_on = False
def showturtle(self):
self._turtle_on = True
def isvisible(self):
return self._turtle_on
def reset(self):
self._reset()
self._line = 'clear'
class Axis(BaseAxis):
"""A line axis.
A line axis is the visual representation of a numerical or date scale.
Attributes
----------
icon: string (class-level attribute)
The font-awesome icon name for this object.
axis_types: dict (class-level attribute)
A registry of existing axis types.
orientation: {'horizontal', 'vertical'}
The orientation of the axis, either vertical or horizontal
side: {'bottom', 'top', 'left', 'right'} or None (default: None)
The side of the axis, either bottom, top, left or right.
label: string (default: '')
The axis label
tick_format: string or None (default: '')
The tick format for the axis, for dates use d3 string formatting.
scale: Scale
The scale represented by the axis
num_ticks: int or None (default: None)
If tick_values is None, number of ticks
tick_values: numpy.ndarray or None (default: None)
Tick values for the axis
offset: dict (default: {})
Contains a scale and a value {'scale': scale or None,
'value': value of the offset}
If offset['scale'] is None, the corresponding figure scale is used
instead.
label_location: {'middle', 'start', 'end'}
The location of the label along the axis, one of 'start', 'end' or
'middle'
label_color: Color or None (default: None)
The color of the axis label
grid_lines: {'none', 'solid', 'dashed'}
The display of the grid lines
grid_color: Color or None (default: None)
The color of the grid lines
color: Color or None (default: None)
The color of the line
label_offset: string or None (default: None)
Label displacement from the axis line. Units allowed are 'em', 'px'
and 'ex'. Positive values are away from the figure and negative
values are towards the figure with respect to the axis line.
visible: bool (default: True)
A visibility toggle for the axis
tick_style: Dict (default: {})
Dictionary containing the CSS-style of the text for the ticks.
For example: font-size of the text can be changed by passing
`{'font-size': 14}`
tick_rotate: int (default: 0)
Degrees to rotate tick labels by.
"""
icon = 'fa-arrows'
orientation = Enum(['horizontal', 'vertical'], default_value='horizontal')\
.tag(sync=True)
side = Enum(['bottom', 'top', 'left', 'right'],
allow_none=True, default_value=None).tag(sync=True)
label = Unicode().tag(sync=True)
grid_lines = Enum(['none', 'solid', 'dashed'], default_value='solid')\
.tag(sync=True)
tick_format = Unicode(None, allow_none=True).tag(sync=True)
scale = Instance(Scale).tag(sync=True, **widget_serialization)
num_ticks = Int(default_value=None, allow_none=True).tag(sync=True)
tick_values = Array(None, allow_none=True)\
.tag(sync=True, **array_serialization)\
.valid(array_dimension_bounds(1, 1))
offset = Dict().tag(sync=True, **widget_serialization)
label_location = Enum(['middle', 'start', 'end'],
default_value='middle').tag(sync=True)
label_color = Color(None, allow_none=True).tag(sync=True)
grid_color = Color(None, allow_none=True).tag(sync=True)
color = Color(None, allow_none=True).tag(sync=True)
label_offset = Unicode(default_value=None, allow_none=True).tag(sync=True)
visible = Bool(True).tag(sync=True)
tick_style = Dict().tag(sync=True)
tick_rotate = Int(0).tag(sync=True)
_view_name = Unicode('Axis').tag(sync=True)
_model_name = Unicode('AxisModel').tag(sync=True)
_ipython_display_ = None # We cannot display an axis outside of a figure.
class NotebookHTTPPersonality(LoggingConfigurable):
"""Personality for notebook-http support, creating REST endpoints
based on the notebook's annotated cells
"""
cell_parser_env = 'KG_CELL_PARSER'
cell_parser = Unicode(
'kernel_gateway.notebook_http.cell.parser',
config=True,
help=
"""Determines which module is used to parse the notebook for endpoints and
documentation. Valid module names include 'kernel_gateway.notebook_http.cell.parser'
and 'kernel_gateway.notebook_http.swagger.parser'. (KG_CELL_PARSER env var)
""")
@default('cell_parser')
def cell_parser_default(self):
return os.getenv(self.cell_parser_env,
'kernel_gateway.notebook_http.cell.parser')
# Intentionally not defining an env var option for a dict type
comment_prefix = Dict({
'scala': '//',
None: '#'
},
config=True,
help='Maps kernel language to code comment syntax')
allow_notebook_download_env = 'KG_ALLOW_NOTEBOOK_DOWNLOAD'
allow_notebook_download = Bool(
config=True,
help=
"Optional API to download the notebook source code in notebook-http mode, defaults to not allow"
)
@default('allow_notebook_download')
def allow_notebook_download_default(self):
return os.getenv(self.allow_notebook_download_env, 'False') == 'True'
static_path_env = 'KG_STATIC_PATH'
static_path = Unicode(
None,
config=True,
allow_none=True,
help=
"Serve static files on disk in the given path as /public, defaults to not serve"
)
@default('static_path')
def static_path_default(self):
return os.getenv(self.static_path_env)
def __init__(self, *args, **kwargs):
super(NotebookHTTPPersonality, self).__init__(*args, **kwargs)
# Import the module to use for cell endpoint parsing
cell_parser_module = importlib.import_module(self.cell_parser)
# Build the parser using the comment syntax for the notebook language
func = getattr(cell_parser_module, 'create_parser')
try:
lang = self.parent.seed_source['metadata']['language_info']['name']
except (AttributeError, KeyError):
lang = None
prefix = self.comment_prefix[lang]
self.api_parser = func(parent=self,
log=self.log,
comment_prefix=prefix,
notebook_cells=self.parent.seed_notebook.cells)
def init_configurables(self):
"""Create a managed kernel pool"""
self.kernel_pool = ManagedKernelPool(self.parent.prespawn_count,
self.parent.kernel_manager)
def create_request_handlers(self):
"""Create handlers and redefine them off of the base_url path. Assumes
init_configurables() has already been called, and that the seed source
was available there.
"""
handlers = []
# Register the NotebookDownloadHandler if configuration allows
if self.allow_notebook_download:
path = url_path_join('/', self.parent.base_url, r'/_api/source')
self.log.info(
'Registering resource: {}, methods: (GET)'.format(path))
handlers.append((path, NotebookDownloadHandler, {
'path': self.parent.seed_uri
}))
# Register a static path handler if configuration allows
if self.static_path is not None:
path = url_path_join('/', self.parent.base_url, r'/public/(.*)')
self.log.info(
'Registering resource: {}, methods: (GET)'.format(path))
handlers.append((path, tornado.web.StaticFileHandler, {
'path': self.static_path
}))
# Discover the notebook endpoints and their implementations
endpoints = self.api_parser.endpoints(
self.parent.kernel_manager.seed_source)
response_sources = self.api_parser.endpoint_responses(
self.parent.kernel_manager.seed_source)
if len(endpoints) == 0:
raise RuntimeError(
'No endpoints were discovered. Check your notebook to make sure your cells are annotated correctly.'
)
# Cycle through the (endpoint_path, source) tuples and register their handlers
for endpoint_path, verb_source_map in endpoints:
parameterized_path = parameterize_path(endpoint_path)
parameterized_path = url_path_join('/', self.parent.base_url,
parameterized_path)
self.log.info('Registering resource: {}, methods: ({})'.format(
parameterized_path, list(verb_source_map.keys())))
response_source_map = response_sources[
endpoint_path] if endpoint_path in response_sources else {}
handler_args = {
'sources': verb_source_map,
'response_sources': response_source_map,
'kernel_pool': self.kernel_pool,
'kernel_name': self.parent.kernel_manager.seed_kernelspec
}
handlers.append(
(parameterized_path, NotebookAPIHandler, handler_args))
# Register the swagger API spec handler
path = url_path_join('/', self.parent.base_url,
r'/_api/spec/swagger.json')
handlers.append((path, SwaggerSpecHandler, {
'notebook_path': self.parent.seed_uri,
'source_cells': self.parent.seed_notebook.cells,
'cell_parser': self.api_parser
}))
self.log.info('Registering resource: {}, methods: (GET)'.format(path))
# Add the 404 catch-all last
handlers.append(default_base_handlers[-1])
return handlers
def should_seed_cell(self, code):
"""Determines whether the given code cell source should be executed when
seeding a new kernel."""
# seed cells that are uninvolved with the presented API
return (not self.api_parser.is_api_cell(code)
and not self.api_parser.is_api_response_cell(code))
def shutdown(self):
"""Stop all kernels in the pool."""
self.kernel_pool.shutdown()
class RasterLayer(Layer):
_view_name = Unicode('LeafletRasterLayerView').tag(sync=True)
_model_name = Unicode('LeafletRasterLayerModel').tag(sync=True)
opacity = Float(1.0, min=0.0, max=1.0).tag(sync=True)
visible = Bool(True).tag(sync=True)
class OAuthenticator(Authenticator):
"""Base class for OAuthenticators
Subclasses must override:
login_service (string identifying the service provider)
authenticate (method takes one arg - the request handler handling the oauth callback)
"""
login_handler = OAuthLoginHandler
callback_handler = OAuthCallbackHandler
authorize_url = Unicode(
config=True, help="""The authenticate url for initiating oauth""")
@default("authorize_url")
def _authorize_url_default(self):
return os.environ.get("OAUTH2_AUTHORIZE_URL", "")
token_url = Unicode(
config=True,
help=
"""The url retrieving an access token at the completion of oauth""",
)
@default("token_url")
def _token_url_default(self):
return os.environ.get("OAUTH2_TOKEN_URL", "")
userdata_url = Unicode(
config=True,
help=
"""The url for retrieving user data with a completed access token""",
)
@default("userdata_url")
def _userdata_url_default(self):
return os.environ.get("OAUTH2_USERDATA_URL", "")
scope = List(
Unicode(),
config=True,
help="""The OAuth scopes to request.
See the OAuth documentation of your OAuth provider for options.
For GitHub in particular, you can see github_scopes.md in this repo.
""",
)
extra_authorize_params = Dict(
config=True,
help="""Extra GET params to send along with the initial OAuth request
to the OAuth provider.""",
)
login_service = 'override in subclass'
oauth_callback_url = Unicode(
os.getenv('OAUTH_CALLBACK_URL', ''),
config=True,
help="""Callback URL to use.
Typically `https://{host}/hub/oauth_callback`""",
)
client_id_env = ''
client_id = Unicode(config=True)
def _client_id_default(self):
if self.client_id_env:
client_id = os.getenv(self.client_id_env, '')
if client_id:
return client_id
return os.getenv('OAUTH_CLIENT_ID', '')
client_secret_env = ''
client_secret = Unicode(config=True)
def _client_secret_default(self):
if self.client_secret_env:
client_secret = os.getenv(self.client_secret_env, '')
if client_secret:
return client_secret
return os.getenv('OAUTH_CLIENT_SECRET', '')
validate_server_cert_env = 'OAUTH_TLS_VERIFY'
validate_server_cert = Bool(config=True)
def _validate_server_cert_default(self):
env_value = os.getenv(self.validate_server_cert_env, '')
if env_value == '0':
return False
else:
return True
def login_url(self, base_url):
return url_path_join(base_url, 'oauth_login')
def get_callback_url(self, handler=None):
"""Get my OAuth redirect URL
Either from config or guess based on the current request.
"""
if self.oauth_callback_url:
return self.oauth_callback_url
elif handler:
return guess_callback_uri(
handler.request.protocol,
handler.request.host,
handler.hub.server.base_url,
)
else:
raise ValueError(
"Specify callback oauth_callback_url or give me a handler to guess with"
)
def get_handlers(self, app):
return [
(r'/oauth_login', self.login_handler),
(r'/oauth_callback', self.callback_handler),
]
async def authenticate(self, handler, data=None):
raise NotImplementedError()
class Map(DOMWidget, InteractMixin):
_view_name = Unicode('LeafletMapView').tag(sync=True)
_model_name = Unicode('LeafletMapModel').tag(sync=True)
_view_module = Unicode('jupyter-leaflet').tag(sync=True)
_model_module = Unicode('jupyter-leaflet').tag(sync=True)
_view_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
_model_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
# Map options
center = List(def_loc).tag(sync=True, o=True)
zoom_start = Int(12).tag(sync=True, o=True)
zoom = Int(12).tag(sync=True, o=True)
max_zoom = Int(18).tag(sync=True, o=True)
min_zoom = Int(1).tag(sync=True, o=True)
interpolation = Unicode('bilinear').tag(sync=True, o=True)
# Specification of the basemap
basemap = Dict(default_value=dict(
url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
max_zoom=19,
attribution=
'Map data (c) <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'
)).tag(sync=True, o=True)
modisdate = Unicode('yesterday').tag(sync=True)
# Interaction options
dragging = Bool(True).tag(sync=True, o=True)
touch_zoom = Bool(True).tag(sync=True, o=True)
scroll_wheel_zoom = Bool(False).tag(sync=True, o=True)
double_click_zoom = Bool(True).tag(sync=True, o=True)
box_zoom = Bool(True).tag(sync=True, o=True)
tap = Bool(True).tag(sync=True, o=True)
tap_tolerance = Int(15).tag(sync=True, o=True)
world_copy_jump = Bool(False).tag(sync=True, o=True)
close_popup_on_click = Bool(True).tag(sync=True, o=True)
bounce_at_zoom_limits = Bool(True).tag(sync=True, o=True)
keyboard = Bool(True).tag(sync=True, o=True)
keyboard_pan_offset = Int(80).tag(sync=True, o=True)
keyboard_zoom_offset = Int(1).tag(sync=True, o=True)
inertia = Bool(True).tag(sync=True, o=True)
inertia_deceleration = Int(3000).tag(sync=True, o=True)
inertia_max_speed = Int(1500).tag(sync=True, o=True)
# inertia_threshold = Int(?, o=True).tag(sync=True)
zoom_control = Bool(True).tag(sync=True, o=True)
attribution_control = Bool(True).tag(sync=True, o=True)
# fade_animation = Bool(?).tag(sync=True, o=True)
# zoom_animation = Bool(?).tag(sync=True, o=True)
zoom_animation_threshold = Int(4).tag(sync=True, o=True)
# marker_zoom_animation = Bool(?).tag(sync=True, o=True)
options = List(trait=Unicode).tag(sync=True)
@default('options')
def _default_options(self):
return [name for name in self.traits(o=True)]
south = Float(def_loc[0], read_only=True).tag(sync=True)
north = Float(def_loc[0], read_only=True).tag(sync=True)
east = Float(def_loc[1], read_only=True).tag(sync=True)
west = Float(def_loc[1], read_only=True).tag(sync=True)
layers = Tuple(trait=Instance(Layer)).tag(sync=True,
**widget_serialization)
@default('layers')
def _default_layers(self):
return (basemap_to_tiles(self.basemap, self.modisdate, base=True), )
bounds = Tuple(read_only=True)
bounds_polygon = Tuple(read_only=True)
@observe('south', 'north', 'east', 'west')
def _observe_bounds(self, change):
self.set_trait('bounds',
((self.south, self.west), (self.north, self.east)))
self.set_trait('bounds_polygon',
((self.north, self.west), (self.north, self.east),
(self.south, self.east), (self.south, self.west)))
def __init__(self, **kwargs):
super(Map, self).__init__(**kwargs)
self.on_displayed(self._fire_children_displayed)
self.on_msg(self._handle_leaflet_event)
def _fire_children_displayed(self, widget, **kwargs):
for layer in self.layers:
layer._handle_displayed(**kwargs)
for control in self.controls:
control._handle_displayed(**kwargs)
_layer_ids = List()
@validate('layers')
def _validate_layers(self, proposal):
'''Validate layers list.
Makes sure only one instance of any given layer can exist in the
layers list.
'''
self._layer_ids = [l.model_id for l in proposal.value]
if len(set(self._layer_ids)) != len(self._layer_ids):
raise LayerException(
'duplicate layer detected, only use each layer once')
return proposal.value
def add_layer(self, layer):
if isinstance(layer, dict):
layer = basemap_to_tiles(layer)
if layer.model_id in self._layer_ids:
raise LayerException('layer already on map: %r' % layer)
self.layers = tuple([l for l in self.layers] + [layer])
def remove_layer(self, layer):
if layer.model_id not in self._layer_ids:
raise LayerException('layer not on map: %r' % layer)
self.layers = tuple(
[l for l in self.layers if l.model_id != layer.model_id])
def substitute_layer(self, old, new):
if isinstance(new, dict):
new = basemap_to_tiles(new)
if old.model_id not in self._layer_ids:
raise LayerException(
'Could not substitute layer: layer not on map.')
self.layers = tuple(
[new if l.model_id == old.model_id else l for l in self.layers])
def clear_layers(self):
self.layers = ()
controls = Tuple(trait=Instance(Control)).tag(sync=True,
**widget_serialization)
_control_ids = List()
@validate('controls')
def _validate_controls(self, proposal):
'''Validate controls list.
Makes sure only one instance of any given layer can exist in the
controls list.
'''
self._control_ids = [c.model_id for c in proposal.value]
if len(set(self._control_ids)) != len(self._control_ids):
raise ControlException(
'duplicate control detected, only use each control once')
return proposal.value
def add_control(self, control):
if control.model_id in self._control_ids:
raise ControlException('control already on map: %r' % control)
self.controls = tuple([c for c in self.controls] + [control])
def remove_control(self, control):
if control.model_id not in self._control_ids:
raise ControlException('control not on map: %r' % control)
self.controls = tuple(
[c for c in self.controls if c.model_id != control.model_id])
def clear_controls(self):
self.controls = ()
def __iadd__(self, item):
if isinstance(item, Layer):
self.add_layer(item)
elif isinstance(item, Control):
self.add_control(item)
return self
def __isub__(self, item):
if isinstance(item, Layer):
self.remove_layer(item)
elif isinstance(item, Control):
self.remove_control(item)
return self
def __add__(self, item):
if isinstance(item, Layer):
self.add_layer(item)
elif isinstance(item, Control):
self.add_control(item)
return self
# Event handling
_interaction_callbacks = Instance(CallbackDispatcher, ())
def _handle_leaflet_event(self, _, content, buffers):
if content.get('event', '') == 'interaction':
self._interaction_callbacks(**content)
def on_interaction(self, callback, remove=False):
self._interaction_callbacks.register_callback(callback, remove=remove)
class ChargeResolutionGenerator(Tool):
name = "ChargeResolutionGenerator"
description = "Generate the a pickle file of ChargeResolutionFile for " \
"either MC or data files."
telescopes = Int(1,
help='Telescopes to include from the event file. '
'Default = 1').tag(config=True)
output_name = Unicode('charge_resolution',
help='Name of the output charge resolution hdf5 '
'file').tag(config=True)
input_path = Unicode(help='Path to directory containing data').tag(
config=True)
max_events = Int(1,
help='Maximum number of events to use').tag(config=True)
plot_cam = Bool(False,
"enable plotting of individual camera").tag(config=True)
use_true_pe = Bool(False, "Use true mc p.e.").tag(config=True)
calibrator = Unicode(
'HESSIOR1Calibrator',
help='which calibrator to use, default = HESSIOR1Calibrator').tag(
config=True)
aliases = Dict(
dict(input_path='ChargeResolutionGenerator.input_path',
calibrator='ChargeResolutionGenerator.calibrator',
max_events='ChargeResolutionGenerator.max_events',
extractor='ChargeExtractorFactory.product',
window_width='ChargeExtractorFactory.window_width',
t0='ChargeExtractorFactory.t0',
window_shift='ChargeExtractorFactory.window_shift',
sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG',
sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG',
lwt='ChargeExtractorFactory.lwt',
clip_amplitude='CameraDL1Calibrator.clip_amplitude',
radius='CameraDL1Calibrator.radius',
max_pe='ChargeResolutionCalculator.max_pe',
T='ChargeResolutionGenerator.telescopes',
o='ChargeResolutionGenerator.output_name',
plot_cam='ChargeResolutionGenerator.plot_cam',
use_true_pe='ChargeResolutionGenerator.use_true_pe'))
classes = List([
EventSourceFactory, HESSIOEventSource, TargetIOEventSource,
ChargeExtractorFactory, CameraDL1Calibrator,
ChargeResolutionCalculator, CameraCalibrator
])
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.eventsource = None
self.r1 = None
self.dl0 = None
self.dl1 = None
self.calculator = None
self.cal = None
def setup(self):
kwargs = dict(config=self.config, tool=self)
self.dl0 = CameraDL0Reducer(**kwargs)
self.dl1 = CameraDL1Calibrator(**kwargs)
self.cal = CameraCalibrator(r1_product=self.calibrator)
self.calculator = ChargeResolutionCalculator(**kwargs)
def start(self):
run_list = np.loadtxt('%s/runlist.txt' % self.input_path, unpack=True)
plot_cam = False
plot_delay = 0.5
disp = None
if debug:
fig = plt.figure(1)
ax = fig.add_subplot(111)
for n, run in enumerate(run_list[0]):
# TODO remove need for hardcoded file name
if self.calibrator == "TargetIOR1Calibrator":
file_name = "%s/Run%05d_r1.tio" % (self.input_path, int(run))
print(file_name)
elif self.calibrator == "HESSIOR1Calibrator":
file_name = "%s/Run%05d_mc.simtel.gz" % (self.input_path,
int(run))
print(file_name)
try:
source = EventSourceFactory.produce(input_url=file_name,
max_events=self.max_events)
true_pe = []
# lab_pe = []
peds_all = []
for event in tqdm(source):
self.cal.calibrate(event)
self.dl0.reduce(event)
self.dl1.calibrate(event)
input_pe = run_list[2][n]
try:
input_nsb = run_list[5][n]
except IndexError:
print('File has no column for NSB, setting to 0')
input_nsb = 0
if self.plot_cam == True:
if disp is None:
geom = event.inst.subarray.tel[
self.telescopes].camera
disp = CameraDisplay(geom)
disp.add_colorbar()
plt.show(block=False)
im = event.dl1.tel[self.telescopes].image[0]
disp.image = im
plt.pause(plot_delay)
teldata = event.r0.tel[self.telescopes].waveform[0]
peds = teldata[:, 0:10].mean(axis=1)
peds2 = teldata[:, 0:10].std(axis=1)
peds_all.append(teldata[:, 0:90])
# plt.hist(peds,bins=50, alpha=0.4)
# plt.show()
# print(teldata)
# plt.plot(range(len(teldata[100])), teldata[100])
# plt.show()
# exit()
# print(np.mean(peds_all), np.std(peds_all))
# exit()
# true_charge_mc = event.mc.tel[self.telescopes].photo_electron_image
# measured_charge = event.dl1.tel[self.telescopes].image[0]
# true_charge_lab = np.asarray([input_pe]*len(measured_charge))
# true_pe.append(true_charge_mc)
# if self.use_true_pe:
# true_charge=true_charge_mc
# else:
# true_charge=true_charge_lab.astype(int)
#
# self.calculator.add_charges(true_charge, measured_charge)
if debug:
plt.errorbar(input_nsb,
np.mean(peds_all),
np.std(peds_all),
marker='x',
color='k')
# plt.scatter(input_nsb, np.std(peds_all), marker ='x',color='k')
plt.xlabel('Non pulsed background light [GHz]')
plt.ylabel('Pedistal mean')
except FileNotFoundError:
stop = 0
print('file_not_found')
plt.show()
# if debug:
# plt.xscale('log')
# plt.yscale('log')
# plt.plot([0,1000],[0,1000], 'k:')
# plt.xlabel('Input p.e.')
# plt.ylabel('True mc p.e.')
# plt.show()
def finish(self):
out_file = '%s/charge_resolution_test.h5' % self.input_path
self.calculator.save(self.output_name)
class HDFSContentsManager(ContentsManager):
"""A ContentsManager implementation that persists to HDFS."""
root_dir = Unicode(
help="""
The root directory to serve from.
By default this is populated by ``root_dir_template``.
""",
config=True
)
@default('root_dir')
def _default_root_dir(self):
return self.root_dir_template.format(
username=getuser()
)
root_dir_template = Unicode(
default_value="/user/{username}/notebooks",
config=True,
help="""
A template string to populate ``root_dir`` from.
Receive the following format parameters:
- username
"""
)
create_root_dir_on_startup = Bool(
default_value=True,
config=True,
help="Create ``root_dir`` on startup if it doesn't already exist"
)
hdfs_host = Unicode(
default_value="default",
config=True,
help="""
The hostname of the HDFS namenode.
By default this will be inferred from the HDFS configuration files.
"""
)
hdfs_port = Integer(
default_value=0,
config=True,
help="""
The port for the HDFS namenode.
By default this will be inferred from the HDFS configuration files.
"""
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.log.debug("Connecting to HDFS at %s:%d",
self.hdfs_host, self.hdfs_port)
self.fs = hdfs.connect(host=self.hdfs_host, port=self.hdfs_port)
if self.create_root_dir_on_startup:
self.ensure_root_directory()
def ensure_root_directory(self):
self.log.debug("Creating root notebooks directory: %s", self.root_dir)
self.fs.mkdir(self.root_dir)
def _checkpoints_class_default(self):
return HDFSCheckpoints
def info_string(self):
return "Serving notebooks from HDFS directory: %s" % self.root_dir
def infer_type(self, path):
if path.endswith(".ipynb"):
return "notebook"
elif self.fs.isdir(path):
return "directory"
else:
return "file"
def is_hidden(self, path):
hdfs_path = to_fs_path(path, self.root_dir)
return is_hidden(hdfs_path, self.root_dir)
def file_exists(self, path):
hdfs_path = to_fs_path(path, self.root_dir)
return self.fs.isfile(hdfs_path)
def dir_exists(self, path):
hdfs_path = to_fs_path(path, self.root_dir)
return self.fs.isdir(hdfs_path)
def exists(self, path):
hdfs_path = to_fs_path(path, self.root_dir)
return self.fs.exists(hdfs_path)
def _info_and_check_kind(self, path, hdfs_path, kind):
try:
with perm_to_403(path):
info = self.fs.info(hdfs_path)
except ArrowIOError:
raise HTTPError(404, "%s does not exist: %s"
% (kind.capitalize(), path))
if info['kind'] != kind:
raise HTTPError(400, "%s is not a %s" % (path, kind))
return info
def _model_from_info(self, info, type=None):
if 'name' in info:
hdfs_path = info['name']
timestamp = info['last_modified_time']
else:
# info from `ls` is different for some reason
hdfs_path = urlsplit(info['path']).path
timestamp = info['last_modified']
path = to_api_path(hdfs_path, self.root_dir)
name = path.rsplit('/', 1)[-1]
if type is None:
if info['kind'] == 'directory':
type = 'directory'
elif path.endswith('.ipynb'):
type = 'notebook'
else:
type = 'file'
mimetype = mimetypes.guess_type(path)[0] if type == 'file' else None
size = info['size'] if type != 'directory' else None
timestamp = utcfromtimestamp(timestamp)
model = {'name': name,
'path': path,
'last_modified': timestamp,
'created': timestamp,
'type': type,
'size': size,
'mimetype': mimetype,
'content': None,
'format': None,
'writable': True}
return model
def _dir_model(self, path, hdfs_path, content):
info = self._info_and_check_kind(path, hdfs_path, 'directory')
model = self._model_from_info(info, 'directory')
if content:
with perm_to_403(path):
records = self.fs.ls(hdfs_path, True)
contents = [self._model_from_info(i) for i in records]
# Filter out hidden files/directories
# These are rare, so do this after generating contents, not before
model['content'] = [c for c in contents
if self.should_list(c['name']) and not
c['name'].startswith('.')]
model['format'] = 'json'
return model
def _file_model(self, path, hdfs_path, content, format):
info = self._info_and_check_kind(path, hdfs_path, 'file')
model = self._model_from_info(info, 'file')
if content:
content, format = self._read_file(path, hdfs_path, format)
if model['mimetype'] is None:
model['mimetype'] = {
'text': 'text/plain',
'base64': 'application/octet-stream'
}[format]
model.update(
content=content,
format=format,
)
return model
def _notebook_model(self, path, hdfs_path, content=True):
info = self._info_and_check_kind(path, hdfs_path, 'file')
model = self._model_from_info(info, 'notebook')
if content:
contents = self._read_notebook(path, hdfs_path)
self.mark_trusted_cells(contents, path)
model['content'] = contents
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def _read_file(self, path, hdfs_path, format):
if not self.fs.isfile(hdfs_path):
raise HTTPError(400, "Cannot read non-file %s" % path)
with perm_to_403(path):
with self.fs.open(hdfs_path, 'rb') as f:
bcontent = f.read()
if format is None:
try:
return bcontent.decode('utf8'), 'text'
except UnicodeError:
return encodebytes(bcontent).decode('ascii'), 'base64'
elif format == 'text':
try:
return bcontent.decode('utf8'), 'text'
except UnicodeError:
raise HTTPError(400, "%s is not UTF-8 encoded" % path,
reason='bad format')
else:
return encodebytes(bcontent).decode('ascii'), 'base64'
def _read_notebook(self, path, hdfs_path):
with perm_to_403(path):
with self.fs.open(hdfs_path, 'rb') as f:
content = f.read()
try:
return nbformat.reads(content.decode('utf8'), as_version=4)
except Exception as e:
raise HTTPError(400, "Unreadable Notebook: %s\n%r" % (path, e))
def get(self, path, content=True, type=None, format=None):
hdfs_path = to_fs_path(path, self.root_dir)
if not self.fs.exists(hdfs_path):
raise HTTPError(404, 'No such file or directory: %s' % path)
elif not self.allow_hidden and is_hidden(hdfs_path, self.root_dir):
self.log.debug("Refusing to serve hidden directory %r", hdfs_path)
raise HTTPError(404, 'No such file or directory: %s' % path)
if type is None:
type = self.infer_type(hdfs_path)
if type == 'directory':
model = self._dir_model(path, hdfs_path, content)
elif type == 'notebook':
model = self._notebook_model(path, hdfs_path, content)
else:
model = self._file_model(path, hdfs_path, content, format)
return model
def _save_directory(self, path, hdfs_path, model):
if not self.allow_hidden and is_hidden(hdfs_path, self.root_dir):
raise HTTPError(400, 'Cannot create hidden directory %r' % path)
if not self.fs.exists(hdfs_path):
self.log.debug("Creating directory at %s", hdfs_path)
with perm_to_403(path):
self.fs.mkdir(hdfs_path)
elif not self.fs.isdir(hdfs_path):
raise HTTPError(400, 'Not a directory: %s' % path)
def _save_file(self, path, hdfs_path, model):
format = model['format']
content = model['content']
if format not in {'text', 'base64'}:
raise HTTPError(
400,
"Must specify format of file contents as 'text' or 'base64'",
)
try:
if format == 'text':
bcontent = content.encode('utf8')
else:
b64_bytes = content.encode('ascii')
bcontent = decodebytes(b64_bytes)
except Exception as e:
raise HTTPError(400, 'Encoding error saving %s: %s' % (path, e))
self.log.debug("Saving file to %s", hdfs_path)
with perm_to_403(path):
with self.fs.open(hdfs_path, 'wb') as f:
f.write(bcontent)
def _save_notebook(self, path, hdfs_path, model):
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
content = nbformat.writes(nb, version=nbformat.NO_CONVERT)
bcontent = content.encode('utf8')
self.log.debug("Saving notebook to %s", hdfs_path)
with perm_to_403(path):
with self.fs.open(hdfs_path, 'wb') as f:
f.write(bcontent)
self.validate_notebook_model(model)
return model.get('message')
def save(self, model, path):
if 'type' not in model:
raise HTTPError(400, 'No file type provided')
typ = model['type']
if 'content' not in model and typ != 'directory':
raise HTTPError(400, 'No file content provided')
hdfs_path = to_fs_path(path, self.root_dir)
message = None
if typ == 'notebook':
message = self._save_notebook(path, hdfs_path, model)
elif typ == 'file':
self._save_file(path, hdfs_path, model)
elif typ == 'directory':
self._save_directory(path, hdfs_path, model)
else:
raise HTTPError(400, "Unhandled contents type: %s" % typ)
model = self.get(path, type=model["type"], content=False)
if message is not None:
model['message'] = message
return model
def _is_dir_empty(self, path, hdfs_path):
with perm_to_403(path):
files = self.fs.ls(hdfs_path)
if not files:
return True
cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None)
files = {f.rsplit('/', 1)[-1] for f in files} - {cp_dir}
return not files
def delete_file(self, path):
hdfs_path = to_fs_path(path, self.root_dir)
if not self.fs.exists(hdfs_path):
raise HTTPError(
404, 'File or directory does not exist: %s' % path
)
if self.fs.isdir(hdfs_path):
if not self._is_dir_empty(path, hdfs_path):
raise HTTPError(400, 'Directory %s not empty' % path)
self.log.debug("Deleting directory at %s", hdfs_path)
with perm_to_403(path):
self.fs.delete(hdfs_path, recursive=True)
else:
self.log.debug("Deleting file at %s", hdfs_path)
with perm_to_403(path):
self.fs.delete(hdfs_path)
def rename_file(self, old_path, new_path):
if old_path == new_path:
return
old_hdfs_path = to_fs_path(old_path, self.root_dir)
new_hdfs_path = to_fs_path(new_path, self.root_dir)
if self.fs.exists(new_hdfs_path):
raise HTTPError(409, 'File already exists: %s' % new_path)
# Move the file
self.log.debug("Renaming %s -> %s", old_hdfs_path, new_hdfs_path)
try:
with perm_to_403(old_path):
self.fs.rename(old_hdfs_path, new_hdfs_path)
except HTTPError:
raise
except Exception as e:
raise HTTPError(
500, 'Unknown error renaming file: %s\n%s' % (old_path, e)
)
