class ImageButton(Button):
"""An example widget."""
_view_name = Unicode("ImageButtonView").tag(sync=True)
_model_name = Unicode("ImageButtonModel").tag(sync=True)
_view_module = Unicode("jupyter_cadquery").tag(sync=True)
_model_module = Unicode("jupyter_cadquery").tag(sync=True)
_view_module_version = Unicode("2.0.2").tag(sync=True)
_model_module_version = Unicode("2.0.2").tag(sync=True)
image_path = Unicode("")
value = Bytes().tag(sync=True, **bytes_serialization)
width = CUnicode("36", help="Width of the image in pixels.").tag(sync=True)
height = CUnicode("28",
help="Height of the image in pixels.").tag(sync=True)
type = Unicode("").tag(sync=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.value = self._load_image(self.image_path)
def _load_image(self, image_path):
if image_path == "":
return b""
else:
return open(image_path, "rb").read()
class Video(_Media):
"""Displays a video as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw video data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "mp4").
If you pass `"url"` to the `"format"` trait, `value` will be interpreted
as a URL as bytes encoded in UTF-8.
"""
_view_name = Unicode('VideoView').tag(sync=True)
_model_name = Unicode('VideoModel').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('mp4', help="The format of the video.").tag(sync=True)
width = CUnicode(help="Width of the video in pixels.").tag(sync=True)
height = CUnicode(help="Height of the video in pixels.").tag(sync=True)
autoplay = Bool(True, help="When true, the video starts when it's displayed").tag(sync=True)
loop = Bool(True, help="When true, the video will start from the beginning after finishing").tag(sync=True)
controls = Bool(True, help="Specifies that video controls should be displayed (such as a play/pause button etc)").tag(sync=True)
@classmethod
def from_file(cls, filename, **kwargs):
return cls._from_file('video', filename, **kwargs)
def __repr__(self):
return self._get_repr(Video)
class Image(widgets.DOMWidget):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
_model_module = Unicode('animation-widget').tag(sync=True)
_view_module = Unicode('animation-widget').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png').tag(sync=True)
width = CUnicode().tag(sync=True)
height = CUnicode().tag(sync=True)
_b64value = Unicode().tag(sync=True)
value = Bytes()
def _value_changed(self, name, old, new):
"""
Service that change the base64 value of image
"""
self._b64value = base64.b64encode(new)
class ImageButton(DOMWidget):
disabled = Bool(False, help="Enable or disable user changes.", sync=True)
_view_name = Unicode('ImageButtonView', sync=True)
format = Unicode('png', sync=True)
width = CUnicode(sync=True)
height = CUnicode(sync=True)
_b64value = Unicode(sync=True)
value = Bytes()
def _value_changed(self, name, old, new):
self._b64value = base64.b64encode(new)
def __init__(self, **kwargs):
super(ImageButton, self).__init__(**kwargs)
self._click_handlers = CallbackDispatcher()
self.on_msg(self._handle_button_msg)
def on_click(self, callback, remove=False):
self._click_handlers.register_callback(callback, remove=remove)
def _handle_button_msg(self, _, content):
if content.get('event', '') == 'click':
self._click_handlers(self, content)
class Image(_Media):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
If you pass `"url"` to the `"format"` trait, `value` will be interpreted
as a URL as bytes encoded in UTF-8.
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png', help="The format of the image.").tag(sync=True)
width = CUnicode(help="Width of the image in pixels. Use layout.width "
"for styling the widget.").tag(sync=True)
height = CUnicode(help="Height of the image in pixels. Use layout.height "
"for styling the widget.").tag(sync=True)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@classmethod
def from_file(cls, filename, **kwargs):
return cls._from_file('image', filename, **kwargs)
def __repr__(self):
return self._get_repr(Image)
class Image(DOMWidget, ValueWidget, CoreWidget):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png', help="The format of the image.").tag(sync=True)
width = CUnicode(help="Width of the image in pixels.").tag(sync=True)
height = CUnicode(help="Height of the image in pixels.").tag(sync=True)
value = Bytes(help="The image data as a byte string.").tag(sync=True)
class DOMWidget(Widget):
visible = Bool(True, allow_none=True, help="Whether the widget is visible. False collapses the empty space, while None preserves the empty space.", sync=True)
_css = Tuple(sync=True, help="CSS property list: (selector, key, value)")
_dom_classes = Tuple(sync=True, help="DOM classes applied to widget.$el.")
width = CUnicode(sync=True)
height = CUnicode(sync=True)
# A default padding of 2.5 px makes the widgets look nice when displayed inline.
padding = CUnicode(sync=True)
margin = CUnicode(sync=True)
color = Color(None, allow_none=True, sync=True)
background_color = Color(None, allow_none=True, sync=True)
border_color = Color(None, allow_none=True, sync=True)
border_width = CUnicode(sync=True)
border_radius = CUnicode(sync=True)
border_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_border-style.asp
'none',
'hidden',
'dotted',
'dashed',
'solid',
'double',
'groove',
'ridge',
'inset',
'outset',
'initial',
'inherit', ''],
default_value='', sync=True)
font_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_font-style.asp
'normal',
'italic',
'oblique',
'initial',
'inherit', ''],
default_value='', sync=True)
font_weight = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_weight.asp
'normal',
'bold',
'bolder',
'lighter',
'initial',
'inherit', ''] + list(map(str, range(100,1000,100))),
default_value='', sync=True)
font_size = CUnicode(sync=True)
font_family = Unicode(sync=True)
def __init__(self, *pargs, **kwargs):
super(DOMWidget, self).__init__(*pargs, **kwargs)
def _validate_border(name, old, new):
if new is not None and new != '':
if name != 'border_width' and not self.border_width:
self.border_width = 1
if name != 'border_style' and self.border_style == '':
self.border_style = 'solid'
self.on_trait_change(_validate_border, ['border_width', 'border_style', 'border_color'])
class Image(DOMWidget, ValueWidget, CoreWidget):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
_model_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_module = Unicode('jupyter-js-widgets').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png').tag(sync=True)
width = CUnicode().tag(sync=True)
height = CUnicode().tag(sync=True)
_b64value = Unicode().tag(sync=True)
value = Bytes()
@observe('value')
def _value_changed(self, change):
self._b64value = base64.b64encode(change['new'])
class BaseParallelApplication(BaseIPythonApplication):
"""The base Application for ipyparallel apps
Primary extensions to BaseIPythonApplication:
* work_dir
* remote logging via pyzmq
* IOLoop instance
"""
version = __version__
_deprecated_classes = None
def init_crash_handler(self):
# disable crash handler from IPython
pass
def _log_level_default(self):
# temporarily override default_log_level to INFO
return logging.INFO
def _log_format_default(self):
"""override default log format to include time"""
return "%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
work_dir = Unicode(os.getcwd(),
config=True,
help='Set the working dir for the process.')
@observe('work_dir')
def _work_dir_changed(self, change):
self.work_dir = str(expand_path(change['new']))
log_to_file = Bool(config=True, help="whether to log to a file")
clean_logs = Bool(False,
config=True,
help="whether to cleanup old logfiles before starting")
log_url = Unicode('',
config=True,
help="The ZMQ URL of the iplogger to aggregate logging.")
cluster_id = CUnicode(
'',
config=True,
help=
"""String id to add to runtime files, to prevent name collisions when
using multiple clusters with a single profile simultaneously.
When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
Since this is text inserted into filenames, typical recommendations apply:
Simple character strings are ideal, and spaces are not recommended (but should
generally work).
""",
)
loop = Instance(IOLoop)
def _loop_default(self):
return IOLoop.current()
session = Instance(Session)
@default("session")
def _default_session(self):
return Session(parent=self)
aliases = base_aliases
flags = base_flags
@catch_config_error
def initialize(self, argv=None):
"""initialize the app"""
self.init_config_from_env()
super().initialize(argv)
self.init_deprecated_config()
self.to_work_dir()
self.reinit_logging()
def init_config_from_env(self):
"""Load any configuration from environment variables"""
if "IPP_SESSION_KEY" in os.environ:
self.config.Session.key = os.environ["IPP_SESSION_KEY"].encode(
"ascii")
if "IPP_CLUSTER_ID" in os.environ:
self.cluster_id = os.environ["IPP_CLUSTER_ID"]
if "IPP_PROFILE_DIR" in os.environ:
self.config.ProfileDir.location = os.environ["IPP_PROFILE_DIR"]
def init_deprecated_config(self):
if not self._deprecated_classes:
return
deprecated_config_found = False
new_classname = self.__class__.__name__
for deprecated_classname in self._deprecated_classes:
if deprecated_classname in self.config:
cfg = self.config[deprecated_classname]
new_config = self.config[new_classname]
for key, deprecated_value in list(cfg.items()):
if key in new_config:
new_value = new_config[key]
if new_value != deprecated_value:
self.log.warning(
f"Ignoring c.{deprecated_classname}.{key} = {deprecated_value}, overridden by c.{new_classname}.{key} = {new_value}"
)
else:
self.log.warning(
f"c.{deprecated_classname}.{key} is deprecated in ipyparallel 7, use c.{new_classname}.{key} = {deprecated_value}"
)
new_config[key] = deprecated_value
cfg.pop(key)
deprecated_config_found = True
if deprecated_config_found:
# reload config
self.update_config(self.config)
def to_work_dir(self):
wd = self.work_dir
if wd != os.getcwd():
os.chdir(wd)
self.log.info("Changing to working dir: %s" % wd)
# This is the working dir by now.
sys.path.insert(0, '')
def reinit_logging(self):
# Remove old log files
log_dir = self.profile_dir.log_dir
if self.clean_logs:
for f in os.listdir(log_dir):
if re.match(r'%s-\d+\.(log|err|out)' % self.name, f):
try:
os.remove(os.path.join(log_dir, f))
except OSError:
# probably just conflict from sibling process
# already removing it
pass
if self.log_to_file:
# Start logging to the new log file
log_filename = f"{self.name}-{self.cluster_id}-{os.getpid()}.log"
logfile = os.path.join(log_dir, log_filename)
if sys.__stderr__:
print(f"Sending logs to {logfile}", file=sys.__stderr__)
open_log_file = open(logfile, 'w')
else:
open_log_file = None
if open_log_file is not None:
while self.log.handlers:
self.log.removeHandler(self.log.handlers[0])
self._log_handler = logging.StreamHandler(open_log_file)
self.log.addHandler(self._log_handler)
else:
self._log_handler = self.log.handlers[0]
# Add timestamps to log format:
self._log_formatter = LevelFormatter(self.log_format,
datefmt=self.log_datefmt)
self._log_handler.setFormatter(self._log_formatter)
# do not propagate log messages to root logger
# ipcluster app will sometimes print duplicate messages during shutdown
# if this is 1 (default):
self.log.propagate = False
class ComponentSync(widgets.Widget):
componentType = Unicode('componentType').tag(sync=True)
model = Unicode('').tag(sync=True)
id = Unicode('').tag(sync=True)
value = CUnicode().tag(sync=True)
widget_name = Unicode('').tag(sync=True)
embedded = Bool(True).tag(sync=True)
_model_name = Unicode('ComponentSync').tag(sync=True)
_model_module = Unicode('jupyter_geppetto').tag(sync=True)
read_only = Bool(False).tag(sync=True)
extraData = None
def __init__(self, **kwargs):
super(ComponentSync, self).__init__(**kwargs)
if 'model' in kwargs and kwargs["model"] is not None and kwargs["model"] != '':
synched_models[kwargs["model"]] = self
self._value_handler = widgets.CallbackDispatcher()
#the method updateModel is executed in response to the sync_value event
self._value_handler.register_callback(self.updateModel)
self.on_msg(self._handle_component_msg)
def _handle_component_msg(self, _, content, buffers):
if content.get('event', '') == 'sync_value':
self._value_handler(self, content)
def updateModel(self, *args):
if self.model != None and self.model != '' and args[1]['value'] != None:
try:
value = json.loads(args[1]['value'])
if isinstance(value, (str, unicode)):
value = "'" + value + "'"
else:
value = str(value)
logging.debug("Updating model with new value " + value)
if(args[1]['requirement']):
exec(args[1]['requirement'])
logging.debug("self.model = " + self.model)
exec(self.model + "=" + value)
except Exception as identifier:
logging.exception("Error updating model")
def connect(self):
logging.debug("ComponentSync connecting to " + self.model)
self.send({"type": "connect"})
def disconnect(self):
logging.debug("ComponentSync disconnecting from " + self.model)
self.send({"type": "disconnect"})
self._value_handler.register_callback(self.updateModel, remove=True)
self.on_msg(self._handle_component_msg, remove=True)
def __str__(self):
return "Component Sync => Widget Name: " + self.widget_name + ", Embedded: " + str(self.embedded) + ", Sync Value: " + self.value + ", Model: " + str(self.model) + ", Extra Data: " + self.extraData
class PlainTextFormatter(BaseFormatter):
"""The default pretty-printer.
This uses :mod:`IPython.lib.pretty` to compute the format data of
the object. If the object cannot be pretty printed, :func:`repr` is used.
See the documentation of :mod:`IPython.lib.pretty` for details on
how to write pretty printers. Here is a simple example::
def dtype_pprinter(obj, p, cycle):
if cycle:
return p.text('dtype(...)')
if hasattr(obj, 'fields'):
if obj.fields is None:
p.text(repr(obj))
else:
p.begin_group(7, 'dtype([')
for i, field in enumerate(obj.descr):
if i > 0:
p.text(',')
p.breakable()
p.pretty(field)
p.end_group(7, '])')
"""
# The format type of data returned.
format_type = Unicode('text/plain')
# This subclass ignores this attribute as it always need to return
# something.
enabled = Bool(True).tag(config=False)
max_seq_length = Integer(pretty.MAX_SEQ_LENGTH,
help="""Truncate large collections (lists, dicts, tuples, sets) to this size.
Set to 0 to disable truncation.
"""
).tag(config=True)
# Look for a _repr_pretty_ methods to use for pretty printing.
print_method = ObjectName('_repr_pretty_')
# Whether to pretty-print or not.
pprint = Bool(True).tag(config=True)
# Whether to be verbose or not.
verbose = Bool(False).tag(config=True)
# The maximum width.
max_width = Integer(79).tag(config=True)
# The newline character.
newline = Unicode('\n').tag(config=True)
# format-string for pprinting floats
float_format = Unicode('%r')
# setter for float precision, either int or direct format-string
float_precision = CUnicode('').tag(config=True)
@observe('float_precision')
def _float_precision_changed(self, change):
"""float_precision changed, set float_format accordingly.
float_precision can be set by int or str.
This will set float_format, after interpreting input.
If numpy has been imported, numpy print precision will also be set.
integer `n` sets format to '%.nf', otherwise, format set directly.
An empty string returns to defaults (repr for float, 8 for numpy).
This parameter can be set via the '%precision' magic.
"""
new = change['new']
if '%' in new:
# got explicit format string
fmt = new
try:
fmt%3.14159
except Exception:
raise ValueError("Precision must be int or format string, not %r"%new)
elif new:
# otherwise, should be an int
try:
i = int(new)
assert i >= 0
except ValueError:
raise ValueError("Precision must be int or format string, not %r"%new)
except AssertionError:
raise ValueError("int precision must be non-negative, not %r"%i)
fmt = '%%.%if'%i
if 'numpy' in sys.modules:
# set numpy precision if it has been imported
import numpy
numpy.set_printoptions(precision=i)
else:
# default back to repr
fmt = '%r'
if 'numpy' in sys.modules:
import numpy
# numpy default is 8
numpy.set_printoptions(precision=8)
self.float_format = fmt
# Use the default pretty printers from IPython.lib.pretty.
@default('singleton_printers')
def _singleton_printers_default(self):
return pretty._singleton_pprinters.copy()
@default('type_printers')
def _type_printers_default(self):
d = pretty._type_pprinters.copy()
d[float] = lambda obj,p,cycle: p.text(self.float_format%obj)
return d
@default('deferred_printers')
def _deferred_printers_default(self):
return pretty._deferred_type_pprinters.copy()
#### FormatterABC interface ####
@catch_format_error
def __call__(self, obj):
"""Compute the pretty representation of the object."""
if not self.pprint:
return repr(obj)
else:
# handle str and unicode on Python 2
# io.StringIO only accepts unicode,
# cStringIO doesn't handle unicode on py2,
# StringIO allows str, unicode but only ascii str
stream = pretty.CUnicodeIO()
printer = pretty.RepresentationPrinter(stream, self.verbose,
self.max_width, self.newline,
max_seq_length=self.max_seq_length,
singleton_pprinters=self.singleton_printers,
type_pprinters=self.type_printers,
deferred_pprinters=self.deferred_printers)
printer.pretty(obj)
printer.flush()
return stream.getvalue()
class SingleUserNotebookAppMixin(Configurable):
"""A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
description = dedent("""
Single-user server for JupyterHub. Extends the Jupyter Notebook server.
Meant to be invoked by JupyterHub Spawners, not directly.
""")
examples = ""
subcommands = {}
version = __version__
# must be set in mixin subclass
# make_singleuser_app sets these
# aliases = aliases
# flags = flags
# login_handler_class = JupyterHubLoginHandler
# logout_handler_class = JupyterHubLogoutHandler
# oauth_callback_handler_class = OAuthCallbackHandler
# classes = NotebookApp.classes + [HubOAuth]
# disable single-user app's localhost checking
allow_remote_access = True
# don't store cookie secrets
cookie_secret_file = ''
# always generate a new cookie secret on launch
# ensures that each spawn clears any cookies from previous session,
# triggering OAuth again
cookie_secret = Bytes()
def _cookie_secret_default(self):
return secrets.token_bytes(32)
user = CUnicode().tag(config=True)
group = CUnicode().tag(config=True)
@default('user')
def _default_user(self):
return os.environ.get('JUPYTERHUB_USER') or ''
@default('group')
def _default_group(self):
return os.environ.get('JUPYTERHUB_GROUP') or ''
@observe('user')
def _user_changed(self, change):
self.log.name = change.new
hub_host = Unicode().tag(config=True)
hub_prefix = Unicode('/hub/').tag(config=True)
@default('keyfile')
def _keyfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_KEYFILE') or ''
@default('certfile')
def _certfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_CERTFILE') or ''
@default('client_ca')
def _client_ca_default(self):
return os.environ.get('JUPYTERHUB_SSL_CLIENT_CA') or ''
@default('hub_prefix')
def _hub_prefix_default(self):
base_url = os.environ.get('JUPYTERHUB_BASE_URL') or '/'
return base_url + 'hub/'
hub_api_url = Unicode().tag(config=True)
@default('hub_api_url')
def _hub_api_url_default(self):
return os.environ.get(
'JUPYTERHUB_API_URL') or 'http://127.0.0.1:8081/hub/api'
# defaults for some configurables that may come from service env variables:
@default('base_url')
def _base_url_default(self):
return os.environ.get('JUPYTERHUB_SERVICE_PREFIX') or '/'
# Note: this may be removed if notebook module is >= 5.0.0b1
@validate('base_url')
def _validate_base_url(self, proposal):
"""ensure base_url starts and ends with /"""
value = proposal.value
if not value.startswith('/'):
value = '/' + value
if not value.endswith('/'):
value = value + '/'
return value
@default('port')
def _port_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.port:
return url.port
elif url.scheme == 'http':
return 80
elif url.scheme == 'https':
return 443
return 8888
@default('ip')
def _ip_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.hostname:
return url.hostname
return '127.0.0.1'
# disable some single-user configurables
token = ''
open_browser = False
quit_button = False
trust_xheaders = True
port_retries = (
0 # disable port-retries, since the Spawner will tell us what port to use
)
disable_user_config = Bool(
False,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""",
).tag(config=True)
@validate('notebook_dir')
def _notebook_dir_validate(self, proposal):
value = os.path.expanduser(proposal['value'])
# Strip any trailing slashes
# *except* if it's root
_, path = os.path.splitdrive(value)
if path == os.sep:
return value
value = value.rstrip(os.sep)
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("No such notebook dir: %r" % value)
return value
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
@default('log_format')
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
self.io_loop.add_callback_from_signal(self.io_loop.stop)
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super().migrate_config()
@property
def config_file_paths(self):
path = super().config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super().nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
@validate('static_custom_path')
def _validate_static_custom_path(self, proposal):
path = proposal['value']
if self.disable_user_config:
path = list(_exclude_home(path))
return path
# create dynamic default http client,
# configured with any relevant ssl config
hub_http_client = Any()
@default('hub_http_client')
def _default_client(self):
ssl_context = make_ssl_context(self.keyfile,
self.certfile,
cafile=self.client_ca)
AsyncHTTPClient.configure(None, defaults={"ssl_options": ssl_context})
return AsyncHTTPClient()
async def check_hub_version(self):
"""Test a connection to my Hub
- exit if I can't connect at all
- check version and warn on sufficient mismatch
"""
client = self.hub_http_client
RETRIES = 5
for i in range(1, RETRIES + 1):
try:
resp = await client.fetch(self.hub_api_url)
except Exception:
self.log.exception(
"Failed to connect to my Hub at %s (attempt %i/%i). Is it running?",
self.hub_api_url,
i,
RETRIES,
)
await asyncio.sleep(min(2**i, 16))
else:
break
else:
self.exit(1)
hub_version = resp.headers.get('X-JupyterHub-Version')
_check_version(hub_version, __version__, self.log)
server_name = Unicode()
@default('server_name')
def _server_name_default(self):
return os.environ.get('JUPYTERHUB_SERVER_NAME', '')
hub_activity_url = Unicode(
config=True, help="URL for sending JupyterHub activity updates")
@default('hub_activity_url')
def _default_activity_url(self):
return os.environ.get('JUPYTERHUB_ACTIVITY_URL', '')
hub_activity_interval = Integer(
300,
config=True,
help="""
Interval (in seconds) on which to update the Hub
with our latest activity.
""",
)
@default('hub_activity_interval')
def _default_activity_interval(self):
env_value = os.environ.get('JUPYTERHUB_ACTIVITY_INTERVAL')
if env_value:
return int(env_value)
else:
return 300
_last_activity_sent = Any(allow_none=True)
async def notify_activity(self):
"""Notify jupyterhub of activity"""
client = self.hub_http_client
last_activity = self.web_app.last_activity()
if not last_activity:
self.log.debug("No activity to send to the Hub")
return
if last_activity:
# protect against mixed timezone comparisons
if not last_activity.tzinfo:
# assume naive timestamps are utc
self.log.warning("last activity is using naive timestamps")
last_activity = last_activity.replace(tzinfo=timezone.utc)
if self._last_activity_sent and last_activity < self._last_activity_sent:
self.log.debug("No activity since %s", self._last_activity_sent)
return
last_activity_timestamp = isoformat(last_activity)
async def notify():
self.log.debug("Notifying Hub of activity %s",
last_activity_timestamp)
req = HTTPRequest(
url=self.hub_activity_url,
method='POST',
headers={
"Authorization":
"token {}".format(self.hub_auth.api_token),
"Content-Type": "application/json",
},
body=json.dumps({
'servers': {
self.server_name: {
'last_activity': last_activity_timestamp
}
},
'last_activity': last_activity_timestamp,
}),
)
try:
await client.fetch(req)
except Exception:
self.log.exception("Error notifying Hub of activity")
return False
else:
return True
await exponential_backoff(
notify,
fail_message="Failed to notify Hub of activity",
start_wait=1,
max_wait=15,
timeout=60,
)
self._last_activity_sent = last_activity
async def keep_activity_updated(self):
if not self.hub_activity_url or not self.hub_activity_interval:
self.log.warning("Activity events disabled")
return
self.log.info("Updating Hub with activity every %s seconds",
self.hub_activity_interval)
while True:
try:
await self.notify_activity()
except Exception as e:
self.log.exception("Error notifying Hub of activity")
# add 20% jitter to the interval to avoid alignment
# of lots of requests from user servers
t = self.hub_activity_interval * (1 + 0.2 *
(random.random() - 0.5))
await asyncio.sleep(t)
def initialize(self, argv=None):
# disable trash by default
# this can be re-enabled by config
self.config.FileContentsManager.delete_to_trash = False
return super().initialize(argv)
def start(self):
self.log.info("Starting jupyterhub-singleuser server version %s",
__version__)
# start by hitting Hub to check version
ioloop.IOLoop.current().run_sync(self.check_hub_version)
ioloop.IOLoop.current().add_callback(self.keep_activity_updated)
super().start()
def init_hub_auth(self):
api_token = None
if os.getenv('JPY_API_TOKEN'):
# Deprecated env variable (as of 0.7.2)
api_token = os.environ['JPY_API_TOKEN']
if os.getenv('JUPYTERHUB_API_TOKEN'):
api_token = os.environ['JUPYTERHUB_API_TOKEN']
if not api_token:
self.exit(
"JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?"
)
self.hub_auth = HubOAuth(
parent=self,
api_token=api_token,
api_url=self.hub_api_url,
hub_prefix=self.hub_prefix,
base_url=self.base_url,
keyfile=self.keyfile,
certfile=self.certfile,
client_ca=self.client_ca,
)
# smoke check
if not self.hub_auth.oauth_client_id:
raise ValueError("Missing OAuth client ID")
def init_webapp(self):
# load the hub-related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['log_function'] = log_request
s['user'] = self.user
s['group'] = self.group
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
csp_report_uri = s['csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
headers = s.setdefault('headers', {})
headers['X-JupyterHub-Version'] = __version__
# set CSP header directly to workaround bugs in jupyter/notebook 5.0
headers.setdefault(
'Content-Security-Policy',
';'.join(
["frame-ancestors 'self'", "report-uri " + csp_report_uri]),
)
super().init_webapp()
# add OAuth callback
self.web_app.add_handlers(
r".*$",
[(
urlparse(self.hub_auth.oauth_redirect_uri).path,
self.oauth_callback_handler_class,
)],
)
# apply X-JupyterHub-Version to *all* request handlers (even redirects)
self.patch_default_headers()
self.patch_templates()
def patch_default_headers(self):
if hasattr(RequestHandler, '_orig_set_default_headers'):
return
RequestHandler._orig_set_default_headers = RequestHandler.set_default_headers
def set_jupyterhub_header(self):
self._orig_set_default_headers()
self.set_header('X-JupyterHub-Version', __version__)
RequestHandler.set_default_headers = set_jupyterhub_header
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars['logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
self.jinja_template_vars['hub_host'] = self.hub_host
self.jinja_template_vars['hub_prefix'] = self.hub_prefix
env = self.web_app.settings['jinja2_env']
env.globals['hub_control_panel_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([FunctionLoader(get_page), orig_loader])
class Image(DOMWidget, ValueWidget, CoreWidget):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
If you pass `"url"` to the `"format"` trait, `value` will be interpreted
as a URL as bytes encoded in UTF-8.
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png', help="The format of the image.").tag(sync=True)
width = CUnicode(help="Width of the image in pixels.").tag(sync=True)
height = CUnicode(help="Height of the image in pixels.").tag(sync=True)
value = Bytes(help="The image data as a byte string.").tag(sync=True)
@classmethod
def from_file(cls, filename, **kwargs):
"""
Create an :class:`Image` from a local file.
Parameters
----------
filename: str
The location of a file to read into the value from disk.
**kwargs:
The keyword arguments for `Image`
Returns an `Image` with the value set from the filename.
"""
value = cls._load_file_value(filename)
if 'format' not in kwargs:
img_format = cls._guess_format(filename)
if img_format is not None:
kwargs['format'] = img_format
return cls(value=value, **kwargs)
@classmethod
def from_url(cls, url, **kwargs):
"""
Create an :class:`Image` from a URL.
:code:`Image.from_url(url)` is equivalent to:
.. code-block: python
img = Image(value=url, format='url')
But both unicode and bytes arguments are allowed for ``url``.
Parameters
----------
url: [str, bytes]
The location of a URL to load.
"""
if isinstance(url, _text_type):
# If unicode (str in Python 3), it needs to be encoded to bytes
url = url.encode('utf-8')
return cls(value=url, format='url')
def set_value_from_file(self, filename):
"""
Convenience method for reading a file into `value`.
Parameters
----------
filename: str
The location of a file to read into value from disk.
"""
value = self._load_file_value(filename)
self.value = value
@classmethod
def _load_file_value(cls, filename):
if getattr(filename, 'read', None) is not None:
return filename.read()
else:
with open(filename, 'rb') as f:
return f.read()
@classmethod
def _guess_format(cls, filename):
# file objects may have a .name parameter
name = getattr(filename, 'name', None)
name = name or filename
try:
mtype, _ = mimetypes.guess_type(name)
if not mtype.startswith('image/'):
return None
return mtype[len('image/'):]
except Exception:
return None
def __repr__(self):
# Truncate the value in the repr, since it will
# typically be very, very large.
class_name = self.__class__.__name__
# Return value first like a ValueWidget
signature = []
sig_value = repr(self.value)
prefix, rest = sig_value.split("'", 1)
content = rest[:-1]
if len(content) > 100:
sig_value = "{}'{}...'".format(prefix, content[0:100])
signature.append('%s=%s' % ('value', sig_value))
for key in super(Image, self)._repr_keys():
if key == 'value':
continue
value = str(getattr(self, key))
signature.append('%s=%r' % (key, value))
signature = ', '.join(signature)
return '%s(%s)' % (class_name, signature)
class AirflowPipelineProcessor(RuntimePipelineProcessor):
_type = RuntimeProcessorType.APACHE_AIRFLOW
_name = "airflow"
# Provide users with the ability to identify a writable directory in the
# running container where the notebook | script is executed. The location
# must exist and be known before the container is started.
# Defaults to `/tmp`
WCD = os.getenv("ELYRA_WRITABLE_CONTAINER_DIR", "/tmp").strip().rstrip("/")
# This specifies the default airflow operators included with Elyra. Any Airflow-based
# custom connectors should create/extend the elyra configuration file to include
# those fully-qualified operator/class names.
available_airflow_operators = ListTrait(
CUnicode(),
[
"airflow.operators.slack_operator.SlackAPIPostOperator",
"airflow.operators.bash_operator.BashOperator",
"airflow.operators.email_operator.EmailOperator",
"airflow.operators.http_operator.SimpleHttpOperator",
"airflow.contrib.operators.spark_sql_operator.SparkSqlOperator",
"airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator",
],
help="""List of available Apache Airflow operator names.
Operators available for use within Apache Airflow pipelines. These operators must
be fully qualified (i.e., prefixed with their package names).
""",
).tag(config=True)
# Contains mappings from class to import statement for each available Airflow operator
class_import_map = {}
def __init__(self, root_dir, **kwargs):
super().__init__(root_dir, **kwargs)
if not self.class_import_map: # Only need to load once
for package in self.available_airflow_operators:
parts = package.rsplit(".", 1)
self.class_import_map[
parts[1]] = f"from {parts[0]} import {parts[1]}"
self.log.debug(f"class_package_map = {self.class_import_map}")
def process(self, pipeline):
t0_all = time.time()
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = f"{pipeline.name}-{timestamp}"
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID,
name=pipeline.runtime_config)
api_endpoint = runtime_configuration.metadata.get("api_endpoint")
cos_endpoint = runtime_configuration.metadata.get("cos_endpoint")
cos_bucket = runtime_configuration.metadata.get("cos_bucket")
git_type = SupportedGitTypes.get_instance_by_name(
runtime_configuration.metadata.get("git_type",
SupportedGitTypes.GITHUB.name))
if git_type == SupportedGitTypes.GITLAB and SupportedGitTypes.is_enabled(
SupportedGitTypes.GITLAB) is False:
raise ValueError(
"Python package `python-gitlab` is not installed. "
"Please install using `elyra[gitlab]` to use GitLab as DAG repository."
)
github_api_endpoint = runtime_configuration.metadata.get(
"github_api_endpoint")
github_repo_token = runtime_configuration.metadata.get(
"github_repo_token")
github_repo = runtime_configuration.metadata.get("github_repo")
github_branch = runtime_configuration.metadata.get("github_branch")
self.log_pipeline_info(pipeline_name, "Submitting pipeline")
with tempfile.TemporaryDirectory() as temp_dir:
pipeline_export_path = os.path.join(temp_dir,
f"{pipeline_name}.py")
self.log.debug("Creating temp directory %s", temp_dir)
pipeline_filepath = self.create_pipeline_file(
pipeline=pipeline,
pipeline_export_format="py",
pipeline_export_path=pipeline_export_path,
pipeline_name=pipeline_name,
)
self.log.debug("Uploading pipeline file: %s", pipeline_filepath)
try:
if git_type == SupportedGitTypes.GITHUB:
git_client = GithubClient(server_url=github_api_endpoint,
token=github_repo_token,
repo=github_repo,
branch=github_branch)
else:
git_client = GitLabClient(
server_url=github_api_endpoint,
token=github_repo_token,
project=github_repo,
branch=github_branch,
)
except BaseException as be:
raise RuntimeError(
f"Unable to create a connection to {github_api_endpoint}: {str(be)}"
) from be
git_client.upload_dag(pipeline_filepath, pipeline_name)
self.log.info(
"Waiting for Airflow Scheduler to process and start the pipeline"
)
download_url = git_client.get_git_url(
api_url=github_api_endpoint,
repository_name=github_repo,
repository_branch=github_branch)
self.log_pipeline_info(pipeline_name,
f"pipeline pushed to git: {download_url}",
duration=(time.time() - t0_all))
return AirflowPipelineProcessorResponse(
git_url=f"{download_url}",
run_url=f"{api_endpoint}",
object_storage_url=f"{cos_endpoint}",
object_storage_path=f"/{cos_bucket}/{pipeline_name}",
)
def export(self, pipeline, pipeline_export_format, pipeline_export_path,
overwrite):
# Verify that the AirflowPipelineProcessor supports the given export format
self._verify_export_format(pipeline_export_format)
timestamp = datetime.now().strftime("%m%d%H%M%S")
pipeline_name = f"{pipeline.name}-{timestamp}"
absolute_pipeline_export_path = get_absolute_path(
self.root_dir, pipeline_export_path)
if os.path.exists(absolute_pipeline_export_path) and not overwrite:
raise ValueError("File " + absolute_pipeline_export_path +
" already exists.")
self.log_pipeline_info(
pipeline_name,
f"exporting pipeline as a .{pipeline_export_format} file")
new_pipeline_file_path = self.create_pipeline_file(
pipeline=pipeline,
pipeline_export_format="py",
pipeline_export_path=absolute_pipeline_export_path,
pipeline_name=pipeline_name,
)
return new_pipeline_file_path
def _cc_pipeline(self, pipeline, pipeline_name):
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID,
name=pipeline.runtime_config)
image_namespace = self._get_metadata_configuration(
schemaspace=RuntimeImages.RUNTIME_IMAGES_SCHEMASPACE_ID)
cos_endpoint = runtime_configuration.metadata.get("cos_endpoint")
cos_username = runtime_configuration.metadata.get("cos_username")
cos_password = runtime_configuration.metadata.get("cos_password")
cos_secret = runtime_configuration.metadata.get("cos_secret")
cos_directory = pipeline_name
cos_bucket = runtime_configuration.metadata.get("cos_bucket")
# Create dictionary that maps component Id to its ContainerOp instance
target_ops = []
self.log_pipeline_info(
pipeline_name,
f"processing pipeline dependencies to: {cos_endpoint} "
f"bucket: {cos_bucket} folder: {pipeline_name}",
)
t0_all = time.time()
# Sort operations based on dependency graph (topological order)
sorted_operations = PipelineProcessor._sort_operations(
pipeline.operations)
# Determine whether access to cloud storage is required and check connectivity
for operation in sorted_operations:
if isinstance(operation, GenericOperation):
self._verify_cos_connectivity(runtime_configuration)
break
# All previous operation outputs should be propagated throughout the pipeline.
# In order to process this recursively, the current operation's inputs should be combined
# from its parent's inputs (which, themselves are derived from the outputs of their parent)
# and its parent's outputs.
PipelineProcessor._propagate_operation_inputs_outputs(
pipeline, sorted_operations)
# Scrub all node labels of invalid characters
scrubbed_operations = self._scrub_invalid_characters_from_list(
sorted_operations)
# Generate unique names for all operations
unique_operations = self._create_unique_node_names(scrubbed_operations)
for operation in unique_operations:
if isinstance(operation, GenericOperation):
operation_artifact_archive = self._get_dependency_archive_name(
operation)
self.log.debug(
f"Creating pipeline component:\n {operation} archive : {operation_artifact_archive}"
)
# Collect env variables
pipeline_envs = self._collect_envs(operation,
cos_secret=cos_secret,
cos_username=cos_username,
cos_password=cos_password)
# Generate unique ELYRA_RUN_NAME value and expose it as an
# environment variable in the container.
# Notebook | script nodes are implemented using the kubernetes_pod_operator
# (https://airflow.apache.org/docs/apache-airflow/1.10.12/_api/airflow/contrib/operators/kubernetes_pod_operator/index.html)
# Environment variables that are passed to this operator are
# pre-processed by Airflow at runtime and placeholder values (expressed as '{{ xyz }}'
# - see https://airflow.apache.org/docs/apache-airflow/1.10.12/macros-ref#default-variables)
# replaced.
if pipeline_envs is None:
pipeline_envs = {}
pipeline_envs[
"ELYRA_RUN_NAME"] = f"{pipeline_name}-{{{{ ts_nodash }}}}"
image_pull_policy = None
runtime_image_pull_secret = None
for image_instance in image_namespace:
if image_instance.metadata[
"image_name"] == operation.runtime_image:
if image_instance.metadata.get("pull_policy"):
image_pull_policy = image_instance.metadata[
"pull_policy"]
if image_instance.metadata.get("pull_secret"):
runtime_image_pull_secret = image_instance.metadata[
"pull_secret"]
break
bootscript = BootscriptBuilder(
filename=operation.filename,
cos_endpoint=cos_endpoint,
cos_bucket=cos_bucket,
cos_directory=cos_directory,
cos_dependencies_archive=operation_artifact_archive,
inputs=operation.inputs,
outputs=operation.outputs,
)
target_op = {
"notebook": operation.name,
"id": operation.id,
"argument_list": bootscript.container_cmd,
"runtime_image": operation.runtime_image,
"pipeline_envs": pipeline_envs,
"parent_operation_ids": operation.parent_operation_ids,
"image_pull_policy": image_pull_policy,
"cpu_request": operation.cpu,
"mem_request": operation.memory,
"gpu_limit": operation.gpu,
"operator_source": operation.component_params["filename"],
"is_generic_operator": True,
"doc": operation.doc,
}
if runtime_image_pull_secret is not None:
target_op[
"runtime_image_pull_secret"] = runtime_image_pull_secret
target_ops.append(target_op)
self.log_pipeline_info(
pipeline_name,
f"processing operation dependencies for id: {operation.id}",
operation_name=operation.name,
)
self._upload_dependencies_to_object_store(
runtime_configuration, pipeline_name, operation)
else:
# Retrieve component from cache
component = ComponentCache.instance().get_component(
self._type, operation.classifier)
# Convert the user-entered value of certain properties according to their type
for component_property in component.properties:
# Skip properties for which no value was given
if component_property.ref not in operation.component_params.keys(
):
continue
# Get corresponding property's value from parsed pipeline
property_value_dict = operation.component_params.get(
component_property.ref)
# The type and value of this property can vary depending on what the user chooses
# in the pipeline editor. So we get the current active parameter (e.g. StringControl)
# from the activeControl value
active_property_name = property_value_dict["activeControl"]
# One we have the value (e.g. StringControl) we use can retrieve the value
# assigned to it
property_value = property_value_dict.get(
active_property_name, None)
# If the value is not found, assign it the default value assigned in parser
if property_value is None:
property_value = component_property.value
self.log.debug(
f"Active property name : {active_property_name}, value : {property_value}"
)
self.log.debug(
f"Processing component parameter '{component_property.name}' "
f"of type '{component_property.data_type}'")
if (property_value and str(property_value)[0] == "{"
and str(property_value)[-1] == "}" and isinstance(
json.loads(json.dumps(property_value)), dict)
and set(
json.loads(json.dumps(property_value)).keys())
== {"value", "option"}):
parent_node_name = self._get_node_name(
target_ops,
json.loads(json.dumps(property_value))["value"])
processed_value = "\"{{ ti.xcom_pull(task_ids='" + parent_node_name + "') }}\""
operation.component_params[
component_property.ref] = processed_value
elif component_property.data_type == "boolean":
operation.component_params[
component_property.ref] = property_value
elif component_property.data_type == "string":
# Add surrounding quotation marks to string value for correct rendering
# in jinja DAG template
operation.component_params[
component_property.ref] = json.dumps(
property_value)
elif component_property.data_type == "dictionary":
processed_value = self._process_dictionary_value(
property_value)
operation.component_params[
component_property.ref] = processed_value
elif component_property.data_type == "list":
processed_value = self._process_list_value(
property_value)
operation.component_params[
component_property.ref] = processed_value
# Remove inputs and outputs from params dict until support for data exchange is provided
operation.component_params_as_dict.pop("inputs")
operation.component_params_as_dict.pop("outputs")
# Locate the import statement. If not found raise...
import_stmts = []
# Check for import statement on Component object, otherwise get from class_import_map
import_stmt = component.import_statement or self.class_import_map.get(
component.name)
if import_stmt:
import_stmts.append(import_stmt)
else:
# If we didn't find a mapping to the import statement, let's check if the component
# name includes a package prefix. If it does, log a warning, but proceed, otherwise
# raise an exception.
if len(component.name.split(
".")) > 1: # We (presumably) have a package prefix
self.log.warning(
f"Operator '{component.name}' of node '{operation.name}' is not configured "
f"in the list of available Airflow operators but appears to include a "
f"package prefix and processing will proceed.")
else:
raise ValueError(
f"Operator '{component.name}' of node '{operation.name}' is not configured "
f"in the list of available operators. Please add the fully-qualified "
f"package name for '{component.name}' to the "
f"AirflowPipelineProcessor.available_airflow_operators configuration."
)
target_op = {
"notebook": operation.name,
"id": operation.id,
"imports": import_stmts,
"class_name": component.name,
"parent_operation_ids": operation.parent_operation_ids,
"component_params": operation.component_params_as_dict,
"operator_source": component.component_source,
"is_generic_operator": False,
"doc": operation.doc,
}
target_ops.append(target_op)
ordered_target_ops = OrderedDict()
while target_ops:
for i in range(len(target_ops)):
target_op = target_ops.pop(0)
if not target_op["parent_operation_ids"]:
ordered_target_ops[target_op["id"]] = target_op
self.log.debug("Root Node added : %s",
ordered_target_ops[target_op["id"]])
elif all(deps in ordered_target_ops.keys()
for deps in target_op["parent_operation_ids"]):
ordered_target_ops[target_op["id"]] = target_op
self.log.debug("Dependent Node added : %s",
ordered_target_ops[target_op["id"]])
else:
target_ops.append(target_op)
self.log_pipeline_info(pipeline_name,
"pipeline dependencies processed",
duration=(time.time() - t0_all))
return ordered_target_ops
def create_pipeline_file(self, pipeline, pipeline_export_format,
pipeline_export_path, pipeline_name):
self.log.info("Creating pipeline definition as a ." +
pipeline_export_format + " file")
if pipeline_export_format == "json":
with open(pipeline_export_path, "w", encoding="utf-8") as file:
json.dump(pipeline_export_path,
file,
ensure_ascii=False,
indent=4)
else:
# Load template from installed elyra package
loader = PackageLoader("elyra", "templates/airflow")
template_env = Environment(loader=loader)
template_env.filters[
"regex_replace"] = lambda string: self._scrub_invalid_characters(
string)
template = template_env.get_template("airflow_template.jinja2")
target_ops = self._cc_pipeline(pipeline, pipeline_name)
runtime_configuration = self._get_metadata_configuration(
schemaspace=Runtimes.RUNTIMES_SCHEMASPACE_ID,
name=pipeline.runtime_config)
user_namespace = runtime_configuration.metadata.get(
"user_namespace") or "default"
cos_secret = runtime_configuration.metadata.get("cos_secret")
pipeline_description = pipeline.description
if pipeline_description is None:
pipeline_description = f"Created with Elyra {__version__} pipeline editor using `{pipeline.source}`."
python_output = template.render(
operations_list=target_ops,
pipeline_name=pipeline_name,
namespace=user_namespace,
cos_secret=cos_secret,
kube_config_path=None,
is_paused_upon_creation="False",
in_cluster="True",
pipeline_description=pipeline_description,
)
# Write to python file and fix formatting
with open(pipeline_export_path, "w") as fh:
# Defer the import to postpone logger messages: https://github.com/psf/black/issues/2058
import black
autopep_output = autopep8.fix_code(python_output)
output_to_file = black.format_str(autopep_output,
mode=black.FileMode())
fh.write(output_to_file)
return pipeline_export_path
def _create_unique_node_names(
self, operation_list: List[Operation]) -> List[Operation]:
unique_names = {}
for operation in operation_list:
# Ensure operation name is unique
new_name = operation.name
while new_name in unique_names:
new_name = f"{operation.name}_{unique_names[operation.name]}"
unique_names[operation.name] += 1
operation.name = new_name
unique_names[operation.name] = 1
return operation_list
def _scrub_invalid_characters_from_list(
self, operation_list: List[Operation]) -> List[Operation]:
for operation in operation_list:
operation.name = self._scrub_invalid_characters(operation.name)
return operation_list
def _scrub_invalid_characters(self, name: str) -> str:
chars = re.escape(string.punctuation)
clean_name = re.sub(r"[" + chars + "\\s]", "_", name) # noqa E226
return clean_name
def _process_dictionary_value(self, value: str) -> Union[Dict, str]:
"""
For component parameters of type dictionary, if a string value is returned from the superclass
method, it must be converted to include surrounding quotation marks for correct rendering
in jinja DAG template.
"""
converted_value = super()._process_dictionary_value(value)
if isinstance(converted_value, str):
converted_value = json.dumps(converted_value)
return converted_value
def _process_list_value(self, value: str) -> Union[List, str]:
"""
For component parameters of type list, if a string value is returned from the superclass
method, it must be converted to include surrounding quotation marks for correct rendering
in jinja DAG template.
"""
converted_value = super()._process_list_value(value)
if isinstance(converted_value, str):
converted_value = json.dumps(converted_value)
return converted_value
def _get_node_name(self, operations_list: list,
node_id: str) -> Optional[str]:
for operation in operations_list:
if operation["id"] == node_id:
return operation["notebook"]
return None
class SingleUserNotebookApp(NotebookApp):
"""A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
description = dedent("""
Single-user server for JupyterHub. Extends the Jupyter Notebook server.
Meant to be invoked by JupyterHub Spawners, and not directly.
""")
examples = ""
subcommands = {}
version = __version__
classes = NotebookApp.classes + [HubOAuth]
user = CUnicode().tag(config=True)
group = CUnicode().tag(config=True)
@observe('user')
def _user_changed(self, change):
self.log.name = change.new
hub_host = Unicode().tag(config=True)
hub_prefix = Unicode('/hub/').tag(config=True)
@default('hub_prefix')
def _hub_prefix_default(self):
base_url = os.environ.get('JUPYTERHUB_BASE_URL') or '/'
return base_url + 'hub/'
hub_api_url = Unicode().tag(config=True)
@default('hub_api_url')
def _hub_api_url_default(self):
return os.environ.get(
'JUPYTERHUB_API_URL') or 'http://127.0.0.1:8081/hub/api'
# defaults for some configurables that may come from service env variables:
@default('base_url')
def _base_url_default(self):
return os.environ.get('JUPYTERHUB_SERVICE_PREFIX') or '/'
#Note: this may be removed if notebook module is >= 5.0.0b1
@validate('base_url')
def _validate_base_url(self, proposal):
"""ensure base_url starts and ends with /"""
value = proposal.value
if not value.startswith('/'):
value = '/' + value
if not value.endswith('/'):
value = value + '/'
return value
@default('cookie_name')
def _cookie_name_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_NAME'):
# if I'm a service, use the services cookie name
return 'jupyterhub-services'
@default('port')
def _port_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
return url.port
@default('ip')
def _ip_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
return url.hostname
aliases = aliases
flags = flags
# disble some single-user configurables
token = ''
open_browser = False
trust_xheaders = True
login_handler_class = JupyterHubLoginHandler
logout_handler_class = JupyterHubLogoutHandler
port_retries = 0 # disable port-retries, since the Spawner will tell us what port to use
disable_user_config = Bool(
False,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""").tag(config=True)
@validate('notebook_dir')
def _notebook_dir_validate(self, proposal):
value = os.path.expanduser(proposal['value'])
# Strip any trailing slashes
# *except* if it's root
_, path = os.path.splitdrive(value)
if path == os.sep:
return value
value = value.rstrip(os.sep)
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("No such notebook dir: %r" % value)
return value
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
@default('log_format')
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
ioloop.IOLoop.instance().stop()
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super(SingleUserNotebookApp, self).migrate_config()
@property
def config_file_paths(self):
path = super(SingleUserNotebookApp, self).config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super(SingleUserNotebookApp, self).nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
@validate('static_custom_path')
def _validate_static_custom_path(self, proposal):
path = proposal['value']
if self.disable_user_config:
path = list(_exclude_home(path))
return path
def start(self):
super(SingleUserNotebookApp, self).start()
def init_hub_auth(self):
api_token = None
if os.getenv('JPY_API_TOKEN'):
# Deprecated env variable (as of 0.7.2)
api_token = os.environ['JPY_API_TOKEN']
if os.getenv('JUPYTERHUB_API_TOKEN'):
api_token = os.environ['JUPYTERHUB_API_TOKEN']
if not api_token:
self.exit(
"JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?"
)
self.hub_auth = HubOAuth(
parent=self,
api_token=api_token,
api_url=self.hub_api_url,
hub_prefix=self.hub_prefix,
base_url=self.base_url,
)
def init_webapp(self):
# load the hub-related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['user'] = self.user
s['group'] = self.group
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
s['csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
super(SingleUserNotebookApp, self).init_webapp()
# add OAuth callback
self.web_app.add_handlers(r".*$", [(urlparse(
self.hub_auth.oauth_redirect_uri).path, OAuthCallbackHandler)])
self.patch_templates()
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars['logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
self.jinja_template_vars['hub_host'] = self.hub_host
self.jinja_template_vars['hub_prefix'] = self.hub_prefix
env = self.web_app.settings['jinja2_env']
env.globals['hub_control_panel_url'] = \
self.hub_host + url_path_join(self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([
FunctionLoader(get_page),
orig_loader,
])
class DOMWidget(Widget):
"""Widget that can be inserted into the DOM"""
_model_name = Unicode('DOMWidgetModel',
help="""Name of the backbone model
registered in the front-end to create and sync this widget with.""",
sync=True)
visible = Bool(
True,
allow_none=True,
help=
"Whether the widget is visible. False collapses the empty space, while None preserves the empty space.",
sync=True)
_css = Tuple(sync=True, help="CSS property list: (selector, key, value)")
_dom_classes = Tuple(sync=True, help="DOM classes applied to widget.$el.")
style = Instance(Style, allow_none=True, sync=True, **widget_serialization)
def _style_default(self):
return Style()
width = CUnicode(sync=True)
height = CUnicode(sync=True)
padding = CUnicode(sync=True)
margin = CUnicode(sync=True)
color = Color(None, allow_none=True, sync=True)
background_color = Color(None, allow_none=True, sync=True)
border_color = Color(None, allow_none=True, sync=True)
border_width = CUnicode(sync=True)
border_radius = CUnicode(sync=True)
border_style = CaselessStrEnum(
values=[ # http://www.w3schools.com/cssref/pr_border-style.asp
'none', 'hidden', 'dotted', 'dashed', 'solid', 'double', 'groove',
'ridge', 'inset', 'outset', 'initial', 'inherit', ''
],
default_value='',
sync=True)
font_style = CaselessStrEnum(
values=[ # http://www.w3schools.com/cssref/pr_font_font-style.asp
'normal', 'italic', 'oblique', 'initial', 'inherit', ''
],
default_value='',
sync=True)
font_weight = CaselessStrEnum(
values=[ # http://www.w3schools.com/cssref/pr_font_weight.asp
'normal', 'bold', 'bolder', 'lighter', 'initial', 'inherit', ''
] + list(map(str, range(100, 1000, 100))),
default_value='',
sync=True)
font_size = CUnicode(sync=True)
font_family = Unicode(sync=True)
def __init__(self, *pargs, **kwargs):
super(DOMWidget, self).__init__(*pargs, **kwargs)
def _validate_border(name, old, new):
if new is not None and new != '':
if name != 'border_width' and not self.border_width:
self.border_width = 1
if name != 'border_style' and self.border_style == '':
self.border_style = 'solid'
self.on_trait_change(_validate_border,
['border_width', 'border_style', 'border_color'])
class RepresentationViewer(DOMWidget):
_view_name = Unicode('ChemviewView').tag(sync=True)
_model_name = Unicode('ChemviewModel').tag(sync=True)
_view_module = Unicode('jupyter-widget-chemview').tag(sync=True)
_model_module = Unicode('jupyter-widget-chemview').tag(sync=True)
width = CInt(sync=True)
height = CInt(sync=True)
background = CInt(sync=True)
# Update Camera Hack
camera_str = CUnicode(sync=True)
static_moving = CBool(sync=True)
# Helper
loaded = CBool(False, sync=True)
def __init__(self, width=500, height=500):
'''RepresentationViewer is an IPython notebook widget useful to display 3d scenes through webgl.
Example:
.. code::
from IPython.display import display
rv = RepresentationViewer()
rv.add_representation('point', {'coordinates': coordinates, 'colors': colors, 'sizes': sizes})
display(rv)
.. py:attribute: width
Width in pixels of the IPython widget
.. py:attribute: height
Height in pixels of the IPython widget
.. py:attribute: camera_str
A string-representation of camera position and orientation
.. py:attribute: static_moving
Set to True to make the camera lose the "bouncy" rotation.
'''
super(RepresentationViewer, self).__init__()
self.displayed = False
self.width = width
self.height = height
# Store the events sent from the javascript side
self._event_handlers = defaultdict(list)
# What to do when we export
def callback(content):
display(Image(url=content.get('dataUrl')))
self._connect_event('displayImg', callback)
# A record of the new representations
self.representations = {}
# Things to be called when the js part is done loading
self._displayed_callbacks = []
def on_loaded(name, old, new):
for cb in self._displayed_callbacks:
cb(self)
self.on_trait_change(on_loaded, "loaded")
def add_representation(self, rep_type, options, rep_id=None):
'''Add a 3D representation to the viewer. See User Guide for
a complete description of the representations available.
:return: An unique hexadecimal identifier for the representation.
:rtype: str
'''
# Add our unique id to be able to refer to the representation
if rep_id is None:
rep_id = uuid4().hex
if rep_type in checkers:
options = checkers[rep_type](options)
self.representations[rep_id] = {'rep_type' : rep_type,
'options': options.copy()}
self._remote_call('addRepresentation', type=rep_type, repId=rep_id, options=options)
return rep_id
def remove_representation(self, rep_id):
'''Remove a representation from the viewer
:param str rep_id: the unique identifier generated by RepresentationViewer.add_representation
'''
self._remote_call('removeRepresentation', repId=rep_id)
del self.representations[rep_id]
def update_representation(self, rep_id, options):
'''Update a representation with new data.
:param str rep_id: the unique identifier returned by RepresentationViewer.add_representation
:param dict options: dictionary containing the updated data.
'''
self.representations[rep_id]['options'].update(options)
rep_type = self.representations[rep_id]["rep_type"]
if rep_type in checkers:
options = checkers[rep_type](options)
self._remote_call('updateRepresentation', repId=rep_id, options=options)
def _connect_event(self, event_name, callback):
'''Respond to an event sent by the Javascript side.
Events available:
- displayImg
- serialize
- fullscreen
'''
self._event_handlers[event_name].append(callback)
def _remote_call(self, method_name, **kwargs):
'''Call a method remotely on the javascript side'''
msg = {}
msg['type'] = 'callMethod'
msg['methodName'] = method_name
msg['args'] = serialize_to_dict(kwargs)
if self.displayed is True:
self.send(msg) # This will be received with View.on_msg
else:
# We should prepare a callback to be
# called when widget is displayed
def callback(widget, msg=msg):
widget.send(msg)
self._displayed_callbacks.append(callback)
def _handle_custom_msg(self, content, buffers=None):
# Handle custom messages sent by the javascript counterpart
event = content.get('event', '')
for cb in self._event_handlers[event]:
cb(content)
def _ipython_display_(self, **kwargs):
super(RepresentationViewer, self)._ipython_display_(**kwargs)
self.displayed = True
def get_scene(self):
'''Return a dictionary that uniquely identifies the scene displayed'''
scene = {}
# Camera
camspec = json.loads(self.camera_str)
location = np.array([camspec['position']['x'],
camspec['position']['y'],
camspec['position']['z']], 'float')
quaternion = np.array([camspec['quaternion']['_x'],
camspec['quaternion']['_y'],
camspec['quaternion']['_z'],
camspec['quaternion']['_w']], 'float')
target = np.array([camspec['target']['x'],
camspec['target']['y'],
camspec['target']['z']], 'float')
scene['camera'] = dict(location=location, quaternion=quaternion,
target=target, vfov=camspec['fov'],
aspect=camspec['aspect'])
# Lights: TODO
scene['lights'] = [ {'position': np.array([2, 4, -3]) * 1000,
'color': 0xffffff },
{'position': np.array([-1, 2, 3]) * 1000,
'color': 0xffffff } ]
# Objects
rep = {k: v.copy() for v in self.representations.items()}
scene['representations'] = [v.update({"id" : k}) for k, v in rep.items()]
scene['representations'] = [item.update({'id'})]
scene['background'] = self.background
return scene
@classmethod
def from_scene(cls, scenedict):
"""Build a representation from scenedict"""
self = cls()
for rep in scenedict["representations"]:
self.add_representation(rep["rep_type"], rep["options"], rep['rep_id'])
return self
def autozoom(self, coordinates):
"""Automatically zoom the scene to enclose *coordinates*.
:param coordinates: array-like of shape (N, 3)
"""
coordinates = np.array(coordinates, dtype='float32')
self._remote_call('zoomInto', coordinates=coordinates)
class SingleUserNotebookApp(NotebookApp):
"""A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
description = dedent("""
Single-user server for JupyterHub. Extends the Jupyter Notebook server.
Meant to be invoked by JupyterHub Spawners, and not directly.
""")
examples = ""
subcommands = {}
version = __version__
classes = NotebookApp.classes + [HubOAuth]
# disable single-user app's localhost checking
allow_remote_access = True
# don't store cookie secrets
cookie_secret_file = ''
# always generate a new cookie secret on launch
# ensures that each spawn clears any cookies from previous session,
# triggering OAuth again
cookie_secret = Bytes()
def _cookie_secret_default(self):
return os.urandom(32)
user = CUnicode().tag(config=True)
group = CUnicode().tag(config=True)
@default('user')
def _default_user(self):
return os.environ.get('JUPYTERHUB_USER') or ''
@default('group')
def _default_group(self):
return os.environ.get('JUPYTERHUB_GROUP') or ''
@observe('user')
def _user_changed(self, change):
self.log.name = change.new
hub_host = Unicode().tag(config=True)
hub_prefix = Unicode('/hub/').tag(config=True)
@default('keyfile')
def _keyfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_KEYFILE') or ''
@default('certfile')
def _certfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_CERTFILE') or ''
@default('client_ca')
def _client_ca_default(self):
return os.environ.get('JUPYTERHUB_SSL_CLIENT_CA') or ''
@default('hub_prefix')
def _hub_prefix_default(self):
base_url = os.environ.get('JUPYTERHUB_BASE_URL') or '/'
return base_url + 'hub/'
hub_api_url = Unicode().tag(config=True)
@default('hub_api_url')
def _hub_api_url_default(self):
return os.environ.get(
'JUPYTERHUB_API_URL') or 'http://127.0.0.1:8081/hub/api'
# defaults for some configurables that may come from service env variables:
@default('base_url')
def _base_url_default(self):
return os.environ.get('JUPYTERHUB_SERVICE_PREFIX') or '/'
#Note: this may be removed if notebook module is >= 5.0.0b1
@validate('base_url')
def _validate_base_url(self, proposal):
"""ensure base_url starts and ends with /"""
value = proposal.value
if not value.startswith('/'):
value = '/' + value
if not value.endswith('/'):
value = value + '/'
return value
@default('port')
def _port_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.port:
return url.port
elif url.scheme == 'http':
return 80
elif url.scheme == 'https':
return 443
return 8888
@default('ip')
def _ip_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.hostname:
return url.hostname
return '127.0.0.1'
aliases = aliases
flags = flags
# disble some single-user configurables
token = ''
open_browser = False
quit_button = False
trust_xheaders = True
login_handler_class = JupyterHubLoginHandler
logout_handler_class = JupyterHubLogoutHandler
port_retries = 0 # disable port-retries, since the Spawner will tell us what port to use
disable_user_config = Bool(
False,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""").tag(config=True)
@validate('notebook_dir')
def _notebook_dir_validate(self, proposal):
value = os.path.expanduser(proposal['value'])
# Strip any trailing slashes
# *except* if it's root
_, path = os.path.splitdrive(value)
if path == os.sep:
return value
value = value.rstrip(os.sep)
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("No such notebook dir: %r" % value)
return value
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
@default('log_format')
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
self.io_loop.add_callback_from_signal(self.io_loop.stop)
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super(SingleUserNotebookApp, self).migrate_config()
@property
def config_file_paths(self):
path = super(SingleUserNotebookApp, self).config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super(SingleUserNotebookApp, self).nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
@validate('static_custom_path')
def _validate_static_custom_path(self, proposal):
path = proposal['value']
if self.disable_user_config:
path = list(_exclude_home(path))
return path
async def check_hub_version(self):
"""Test a connection to my Hub
- exit if I can't connect at all
- check version and warn on sufficient mismatch
"""
ssl_context = make_ssl_context(
self.keyfile,
self.certfile,
cafile=self.client_ca,
)
AsyncHTTPClient.configure(None, defaults={"ssl_options": ssl_context})
client = AsyncHTTPClient()
RETRIES = 5
for i in range(1, RETRIES + 1):
try:
resp = await client.fetch(self.hub_api_url)
except Exception:
self.log.exception(
"Failed to connect to my Hub at %s (attempt %i/%i). Is it running?",
self.hub_api_url, i, RETRIES)
await gen.sleep(min(2**i, 16))
else:
break
else:
self.exit(1)
hub_version = resp.headers.get('X-JupyterHub-Version')
_check_version(hub_version, __version__, self.log)
def initialize(self, argv=None):
# disable trash by default
# this can be re-enabled by config
self.config.FileContentsManager.delete_to_trash = False
return super().initialize(argv)
def start(self):
self.log.info("Starting jupyterhub-singleuser server version %s",
__version__)
# start by hitting Hub to check version
ioloop.IOLoop.current().run_sync(self.check_hub_version)
super(SingleUserNotebookApp, self).start()
def init_hub_auth(self):
api_token = None
if os.getenv('JPY_API_TOKEN'):
# Deprecated env variable (as of 0.7.2)
api_token = os.environ['JPY_API_TOKEN']
if os.getenv('JUPYTERHUB_API_TOKEN'):
api_token = os.environ['JUPYTERHUB_API_TOKEN']
if not api_token:
self.exit(
"JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?"
)
self.hub_auth = HubOAuth(
parent=self,
api_token=api_token,
api_url=self.hub_api_url,
hub_prefix=self.hub_prefix,
base_url=self.base_url,
keyfile=self.keyfile,
certfile=self.certfile,
client_ca=self.client_ca,
)
# smoke check
if not self.hub_auth.oauth_client_id:
raise ValueError("Missing OAuth client ID")
def init_webapp(self):
# load the hub-related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['log_function'] = log_request
s['user'] = self.user
s['group'] = self.group
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
csp_report_uri = s['csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
headers = s.setdefault('headers', {})
headers['X-JupyterHub-Version'] = __version__
# set CSP header directly to workaround bugs in jupyter/notebook 5.0
headers.setdefault(
'Content-Security-Policy', ';'.join([
"frame-ancestors 'self'",
"report-uri " + csp_report_uri,
]))
super(SingleUserNotebookApp, self).init_webapp()
# add OAuth callback
self.web_app.add_handlers(r".*$", [(urlparse(
self.hub_auth.oauth_redirect_uri).path, OAuthCallbackHandler)])
# apply X-JupyterHub-Version to *all* request handlers (even redirects)
self.patch_default_headers()
self.patch_templates()
def patch_default_headers(self):
if hasattr(RequestHandler, '_orig_set_default_headers'):
return
RequestHandler._orig_set_default_headers = RequestHandler.set_default_headers
def set_jupyterhub_header(self):
self._orig_set_default_headers()
self.set_header('X-JupyterHub-Version', __version__)
RequestHandler.set_default_headers = set_jupyterhub_header
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars['logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
self.jinja_template_vars['hub_host'] = self.hub_host
self.jinja_template_vars['hub_prefix'] = self.hub_prefix
env = self.web_app.settings['jinja2_env']
env.globals['hub_control_panel_url'] = \
self.hub_host + url_path_join(self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([
FunctionLoader(get_page),
orig_loader,
])
class Directions(widgets.Widget):
"""
Directions layer.
Add this to a :class:`gmaps.Figure` instance to draw directions.
The directions are requested with the ``DRIVING`` option.
Use the :func:`gmaps.directions_layer` factory function to
instantiate this class, rather than the constructor.
:Examples:
>>> fig = gmaps.figure()
>>> start = (46.2, 6.1)
>>> end = (47.4, 8.5)
>>> waypoints = [(52.37403, 4.88969)]
>>> directions_layer = gmaps.directions_layer(start, end, waypoints)
>>> fig.add_layer(directions_layer)
There is a limitation in the number of waypoints allowed by Google
(currently 23). If it
fails to return directions, a ``DirectionsServiceException`` is raised.
>>> directions_layer = gmaps.Directions(data=data*10)
Traceback (most recent call last):
...
DirectionsServiceException: No directions returned: MAX WAYPOINTS EXCEEDED
:param data: List of (latitude, longitude) pairs denoting a single
point. The first pair denotes the starting point and the last pair
denote the end of the route.
Latitudes are expressed as a float between -90
(corresponding to 90 degrees south) and +90 (corresponding to
90 degrees north). Longitudes are expressed as a float
between -180 (corresponding to 180 degrees west) and 180
(corresponding to 180 degrees east).
:type data: list of tuples of length >= 2
:param avoid_ferries: Avoids ferries where possible.
:type avoid_ferries: bool, optional
:param avoid_highways: Avoids highways where possible.
:type avoid_highways: bool, optional
:param avoid_tolls: Avoids toll roads where possible.
:type avoid_tolls: bool, optional
:param optimize_waypoints: Attempt to re-order the supplied intermediate
waypoints to minimize overall cost of the route.
:type optimize_waypoints: bool, optional
"""
has_bounds = True
_view_name = Unicode("DirectionsLayerView").tag(sync=True)
_view_module = Unicode("jupyter-gmaps").tag(sync=True)
_model_name = Unicode("DirectionsLayerModel").tag(sync=True)
_model_module = Unicode("jupyter-gmaps").tag(sync=True)
data = List(minlen=2).tag(sync=True)
data_bounds = List().tag(sync=True)
avoid_ferries = Bool(default_value=False).tag(sync=True)
avoid_highways = Bool(default_value=False).tag(sync=True)
avoid_tolls = Bool(default_value=False).tag(sync=True)
optimize_waypoints = Bool(default_value=False).tag(sync=True)
layer_status = CUnicode().tag(sync=True)
@validate("data")
def _validate_data(self, proposal):
for point in proposal["value"]:
if not geotraitlets.is_valid_point(point):
raise geotraitlets.InvalidPointException(
"{} is not a valid latitude, longitude pair".format(point))
return proposal["value"]
@observe("data")
def _calc_bounds(self, change):
data = change["new"]
min_latitude = min(row[0] for row in data)
min_longitude = min(row[1] for row in data)
max_latitude = max(row[0] for row in data)
max_longitude = max(row[1] for row in data)
self.data_bounds = [
(min_latitude, min_longitude),
(max_latitude, max_longitude)
]
@observe("layer_status")
def _handle_layer_status(self, change):
if change["new"] != "OK":
raise DirectionsServiceException(
"No directions returned: " + change["new"])
class SingleUserNotebookApp(NotebookApp):
"""A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
user = CUnicode(config=True)
def _user_changed(self, name, old, new):
self.log.name = new
hub_prefix = Unicode(config=True)
hub_host = Unicode(config=True)
hub_api_url = Unicode(config=True)
aliases = aliases
flags = flags
open_browser = False
trust_xheaders = True
login_handler_class = JupyterHubLoginHandler
logout_handler_class = JupyterHubLogoutHandler
port_retries = 0 # disable port-retries, since the Spawner will tell us what port to use
disable_user_config = Bool(
False,
config=True,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""")
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
ioloop.IOLoop.instance().stop()
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super(SingleUserNotebookApp, self).migrate_config()
@property
def config_file_paths(self):
path = super(SingleUserNotebookApp, self).config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super(SingleUserNotebookApp, self).nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
def _static_custom_path_default(self):
path = super(SingleUserNotebookApp, self)._static_custom_path_default()
if self.disable_user_config:
path = list(_exclude_home(path))
return path
def start(self):
super(SingleUserNotebookApp, self).start()
def init_hub_auth(self):
self.hub_auth = HubAuth(
parent=self,
api_token=os.environ.pop('JPY_API_TOKEN'),
api_url=self.hub_api_url,
)
def init_webapp(self):
# load the hub related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['user'] = self.user
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
s['login_url'] = self.hub_host + self.hub_prefix
s['csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
super(SingleUserNotebookApp, self).init_webapp()
self.patch_templates()
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars['logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
env = self.web_app.settings['jinja2_env']
env.globals['hub_control_panel_url'] = \
self.hub_host + url_path_join(self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([
FunctionLoader(get_page),
orig_loader,
])
class Directions(GMapsWidgetMixin, widgets.Widget):
"""
Directions layer.
Add this to a :class:`gmaps.Figure` instance to draw directions.
Use the :func:`gmaps.directions_layer` factory function to
instantiate this class, rather than the constructor.
:Examples:
{examples}
{params}
"""
has_bounds = True
_view_name = Unicode('DirectionsLayerView').tag(sync=True)
_model_name = Unicode('DirectionsLayerModel').tag(sync=True)
start = geotraitlets.Point().tag(sync=True)
end = geotraitlets.Point().tag(sync=True)
waypoints = geotraitlets.LocationArray().tag(sync=True)
data = List(minlen=2, allow_none=True, default_value=None)
data_bounds = List().tag(sync=True)
avoid_ferries = Bool(default_value=False).tag(sync=True)
avoid_highways = Bool(default_value=False).tag(sync=True)
avoid_tolls = Bool(default_value=False).tag(sync=True)
optimize_waypoints = Bool(default_value=False).tag(sync=True)
travel_mode = Enum(ALLOWED_TRAVEL_MODES,
default_value=DEFAULT_TRAVEL_MODE).tag(sync=True)
show_markers = Bool(default_value=True).tag(sync=True)
show_route = Bool(default_value=True).tag(sync=True)
stroke_color = geotraitlets.ColorAlpha(default_value=DEFAULT_STROKE_COLOR,
allow_none=False).tag(sync=True)
stroke_opacity = geotraitlets.Opacity(default_value=0.6,
allow_none=False).tag(sync=True)
stroke_weight = Float(min=0.0, allow_none=False,
default_value=6.0).tag(sync=True)
layer_status = CUnicode().tag(sync=True)
def __init__(self, start=None, end=None, waypoints=None, **kwargs):
if kwargs.get('data') is not None:
_warn_obsolete_data()
# Keep for backwards compatibility with data argument
data = kwargs['data']
waypoints = kwargs.get('waypoints')
if start is None and end is None and waypoints is None:
start, end, waypoints = Directions._destructure_data(data)
kwargs.update(
dict(start=start, end=end, waypoints=waypoints, data=None))
else:
raise ValueError(
'Cannot set both data and one of "start", "end"'
'or "waypoints".')
else:
if waypoints is None:
waypoints = []
kwargs.update(dict(start=start, end=end, waypoints=waypoints))
super(Directions, self).__init__(**kwargs)
@staticmethod
def _destructure_data(data):
start = data[0]
end = data[-1]
waypoints = data[1:-1]
return start, end, waypoints
@validate('waypoints')
def _valid_waypoints(self, proposal):
if proposal['value'] is None:
_warn_obsolete_waypoints()
proposal['value'] = []
return proposal['value']
@observe('data')
def _on_data_change(self, change):
data = change['new']
if data is not None:
_warn_obsolete_data()
with self.hold_trait_notifications():
self.start, self.end, self.waypoints = \
self._destructure_data(data)
@observe('start', 'end', 'waypoints')
def _calc_bounds(self, change):
all_data = [self.start] + self.waypoints + [self.end]
min_latitude = min(row[0] for row in all_data)
min_longitude = min(row[1] for row in all_data)
max_latitude = max(row[0] for row in all_data)
max_longitude = max(row[1] for row in all_data)
self.data_bounds = [(min_latitude, min_longitude),
(max_latitude, max_longitude)]
class DOMWidget(Widget):
"""Widget that can be inserted into the DOM"""
_model_name = Unicode('DOMWidgetModel', help="""Name of the backbone model
registered in the front-end to create and sync this widget with.""", sync=True)
visible = Bool(True, allow_none=True, help="Whether the widget is visible. False collapses the empty space, while None preserves the empty space.", sync=True) # TODO: Deprecated in ipywidgets 5.0
_dom_classes = Tuple(sync=True, help="DOM classes applied to widget.$el.")
layout = Instance(Layout, allow_none=True, sync=True, **widget_serialization)
def _layout_default(self):
return Layout()
width = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
height = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
padding = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
margin = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
color = Color(None, allow_none=True, sync=True) # TODO: Deprecated in ipywidgets 5.0
background_color = Color(None, allow_none=True, sync=True) # TODO: Deprecated in ipywidgets 5.0
border_color = Color(None, allow_none=True, sync=True) # TODO: Deprecated in ipywidgets 5.0
border_width = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
border_radius = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
border_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_border-style.asp # TODO: Deprecated in ipywidgets 5.0
'none',
'hidden',
'dotted',
'dashed',
'solid',
'double',
'groove',
'ridge',
'inset',
'outset',
'initial',
'inherit', ''],
default_value='', sync=True)
font_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_font-style.asp # TODO: Deprecated in ipywidgets 5.0
'normal',
'italic',
'oblique',
'initial',
'inherit', ''],
default_value='', sync=True)
font_weight = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_weight.asp # TODO: Deprecated in ipywidgets 5.0
'normal',
'bold',
'bolder',
'lighter',
'initial',
'inherit', ''] + list(map(str, range(100,1000,100))),
default_value='', sync=True)
font_size = CUnicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
font_family = Unicode(sync=True) # TODO: Deprecated in ipywidgets 5.0
def __init__(self, *pargs, **kwargs):
super(DOMWidget, self).__init__(*pargs, **kwargs)
def _validate_border(name, old, new):
if new is not None and new != '':
if name != 'border_width' and not self.border_width:
self.border_width = 1
if name != 'border_style' and self.border_style == '':
self.border_style = 'solid'
self.on_trait_change(_validate_border, ['border_width', 'border_style', 'border_color'])
# Deprecation added in 5.0. TODO: Remove me and corresponging traits.
self._deprecate_traits(['width', 'height', 'padding', 'margin', 'color',
'background_color', 'border_color', 'border_width', 'border_radius',
'border_style', 'font_style', 'font_weight', 'font_size', 'font_family',
'visible'])
def add_class(self, className):
"""
Adds a class to the top level element of the widget.
Doesn't add the class if it already exists.
"""
if className not in self._dom_classes:
self._dom_classes.append(className)
return self
def remove_class(self, className):
"""
Removes a class from the top level element of the widget.
Doesn't remove the class if it doesn't exist.
"""
if className in self._dom_classes:
self._dom_classes.remove(className)
return self
def _deprecate_traits(self, traits): # TODO: Deprecation added in 5.0. Remove me and corresponging traits.
for trait in traits:
def traitWarn():
warn("%s deprecated" % trait, DeprecationWarning)
self.on_trait_change(traitWarn, trait)
class Image(DOMWidget, ValueWidget, CoreWidget):
"""Displays an image as a widget.
The `value` of this widget accepts a byte string. The byte string is the
raw image data that you want the browser to display. You can explicitly
define the format of the byte string using the `format` trait (which
defaults to "png").
If you pass `"url"` to the `"format"` trait, `value` will be interpreted
as a URL as bytes encoded in UTF-8.
"""
_view_name = Unicode('ImageView').tag(sync=True)
_model_name = Unicode('ImageModel').tag(sync=True)
# Define the custom state properties to sync with the front-end
format = Unicode('png', help="The format of the image.").tag(sync=True)
width = CUnicode(help="Width of the image in pixels.").tag(sync=True)
height = CUnicode(help="Height of the image in pixels.").tag(sync=True)
value = Bytes(help="The image data as a byte string.").tag(sync=True)
@classmethod
def from_file(cls, filename, **kwargs):
"""
Create an :class:`Image` from a local file.
Parameters
----------
filename: str
The location of a file to read into the value from disk.
**kwargs:
The keyword arguments for `Image`
Returns an `Image` with the value set from the filename.
"""
value = cls._load_file_value(filename)
if 'format' not in kwargs:
img_format = cls._guess_format(filename)
if img_format is not None:
kwargs['format'] = img_format
return cls(value=value, **kwargs)
@classmethod
def from_url(cls, url, **kwargs):
"""
Create an :class:`Image` from a URL.
:code:`Image.from_url(url)` is equivalent to:
.. code-block: python
img = Image(value=url, format='url')
But both unicode and bytes arguments are allowed for ``url``.
Parameters
----------
url: [str, bytes]
The location of a URL to load.
"""
if isinstance(url, _text_type):
# If unicode (str in Python 3), it needs to be encoded to bytes
url = url.encode('utf-8')
return cls(value=url, format='url')
def set_value_from_file(self, filename):
"""
Convenience method for reading a file into `value`.
Parameters
----------
filename: str
The location of a file to read into value from disk.
"""
value = self._load_file_value(filename)
self.value = value
@classmethod
def _load_file_value(cls, filename):
if getattr(filename, 'read', None) is not None:
return filename.read()
else:
with open(filename, 'rb') as f:
return f.read()
@classmethod
def _guess_format(cls, filename):
# file objects may have a .name parameter
name = getattr(filename, 'name', None)
name = name or filename
try:
mtype, _ = mimetypes.guess_type(name)
if not mtype.startswith('image/'):
return None
return mtype[len('image/'):]
except Exception:
return None
class Directions(widgets.Widget):
"""
Directions layer.
Add this to a ``Map`` instance to draw directions.
By default, the directions are requested with the DRIVING option.
Data is a list of lat,lon tuples. The first point of the list is passed as
the origin of the itinerary; the last point is passed as the destination of
the itinerary. Other points are passed in order as a list of waypoints.
To set the parameters, pass them to the constructor:
>>> directions_layer = gmaps.Directions(data=data)
Examples:
>>> m = gmaps.Map()
>>> data = [(48.85341, 2.3488), (50.85045, 4.34878), (52.37403, 4.88969)]
>>> directions_layer = gmaps.Directions(data=data)
>>> m.add_layer(directions_layer)
An TraitError is raised if you try to pass less than two points:
>>> directions_layer = gmaps.Directions(data=[(50.0, 4.0])
Traceback (most recent call last):
...
TraitError: The 'data' trait of a Directions instance must be of length 2 <= L <= 9223372036854775807, but a value of [[50.0, 4.0]] was specified.
There is a limitation in the number of waypoints allowed by Google. If it
fails to return directions, a DirectionsServiceException is raised.
>>> directions_layer = gmaps.Directions(data=data*10)
Traceback (most recent call last):
...
DirectionsServiceException: No directions returned: MAX WAYPOINTS EXCEEDED
"""
has_bounds = True
_view_name = Unicode("DirectionsLayerView").tag(sync=True)
_view_module = Unicode("jupyter-gmaps").tag(sync=True)
_model_name = Unicode("DirectionsLayerModel").tag(sync=True)
_model_module = Unicode("jupyter-gmaps").tag(sync=True)
data = List(minlen=2).tag(sync=True)
data_bounds = List().tag(sync=True)
layer_status = CUnicode().tag(sync=True)
@validate("data")
def _validate_data(self, proposal):
assert (len(proposal["value"]) >=
2), "A direction requires at least two points"
for point in proposal["value"]:
if not geotraitlets.is_valid_point(point):
raise InvalidPointException(
"{} is not a valid latitude, longitude pair".format(point))
return proposal["value"]
@observe("data")
def _calc_bounds(self, change):
data = change["new"]
min_latitude = min(row[0] for row in data)
min_longitude = min(row[1] for row in data)
max_latitude = max(row[0] for row in data)
max_longitude = max(row[1] for row in data)
self.data_bounds = [(min_latitude, min_longitude),
(max_latitude, max_longitude)]
@observe("layer_status")
def _handle_layer_status(self, change):
if change["new"] != "OK":
raise DirectionsServiceException("No directions returned: " +
change["new"])
class SingleUserNotebookApp(NotebookApp):
"""A Subclass of the regular NotebookApp that is aware of the parent multiuser context."""
description = dedent("""
Single-user server for JupyterHub. Extends the Jupyter Notebook server.
Meant to be invoked by JupyterHub Spawners, and not directly.
""")
examples = ""
subcommands = {}
version = __version__
classes = NotebookApp.classes + [HubAuth]
user = CUnicode(config=True)
def _user_changed(self, name, old, new):
self.log.name = new
hub_prefix = Unicode().tag(config=True)
hub_host = Unicode().tag(config=True)
hub_api_url = Unicode().tag(config=True)
aliases = aliases
flags = flags
# disble some single-user configurables
token = ''
open_browser = False
trust_xheaders = True
login_handler_class = JupyterHubLoginHandler
logout_handler_class = JupyterHubLogoutHandler
port_retries = 0 # disable port-retries, since the Spawner will tell us what port to use
disable_user_config = Bool(
False,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""").tag(config=True)
@validate('notebook_dir')
def _notebook_dir_validate(self, proposal):
value = os.path.expanduser(proposal['value'])
# Strip any trailing slashes
# *except* if it's root
_, path = os.path.splitdrive(value)
if path == os.sep:
return value
value = value.rstrip(os.sep)
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("No such notebook dir: %r" % value)
return value
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
@default('log_format')
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
ioloop.IOLoop.instance().stop()
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super(SingleUserNotebookApp, self).migrate_config()
@property
def config_file_paths(self):
path = super(SingleUserNotebookApp, self).config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super(SingleUserNotebookApp, self).nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
@validate('static_custom_path')
def _validate_static_custom_path(self, proposal):
path = proposal['value']
if self.disable_user_config:
path = list(_exclude_home(path))
return path
def start(self):
super(SingleUserNotebookApp, self).start()
def init_hub_auth(self):
if not os.environ.get('JPY_API_TOKEN'):
self.exit(
"JPY_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?"
)
self.hub_auth = HubAuth(
parent=self,
api_token=os.environ.pop('JPY_API_TOKEN'),
api_url=self.hub_api_url,
)
def init_webapp(self):
# load the hub related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['user'] = self.user
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
s['login_url'] = self.hub_host + self.hub_prefix
s['csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
super(SingleUserNotebookApp, self).init_webapp()
self.patch_templates()
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars['logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
self.jinja_template_vars['hub_host'] = self.hub_host
self.jinja_template_vars['hub_prefix'] = self.hub_prefix
env = self.web_app.settings['jinja2_env']
env.globals['hub_control_panel_url'] = \
self.hub_host + url_path_join(self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([
FunctionLoader(get_page),
orig_loader,
])
class Layout(Widget):
"""Layout specification
Defines a layout that can be expressed using CSS. Supports a subset of
https://developer.mozilla.org/en-US/docs/Web/CSS/Reference
When a property is also accessible via a shorthand property, we only
expose the shorthand.
For example:
- ``flex-grow``, ``flex-shrink`` and ``flex-basis`` are bound to ``flex``.
- ``flex-wrap`` and ``flex-direction`` are bound to ``flex-flow``.
- ``margin-[top/bottom/left/right]`` values are bound to ``margin``, etc.
"""
_model_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_name = Unicode('LayoutView').tag(sync=True)
_model_name = Unicode('LayoutModel').tag(sync=True)
# Keys
align_content = CUnicode().tag(sync=True)
align_items = CUnicode().tag(sync=True)
align_self = CUnicode().tag(sync=True)
bottom = CUnicode().tag(sync=True)
border = CUnicode().tag(sync=True)
display = CUnicode().tag(sync=True)
flex = CUnicode().tag(sync=True)
flex_flow = CUnicode().tag(sync=True)
height = CUnicode().tag(sync=True)
justify_content = CUnicode().tag(sync=True)
left = CUnicode().tag(sync=True)
margin = CUnicode().tag(sync=True)
overflow = CUnicode().tag(sync=True)
padding = CUnicode().tag(sync=True)
right = CUnicode().tag(sync=True)
top = CUnicode().tag(sync=True)
visibility = CUnicode().tag(sync=True)
width = CUnicode().tag(sync=True)
class Session(Configurable):
"""Object for handling serialization and sending of messages.
The Session object handles building messages and sending them
with ZMQ sockets or ZMQStream objects. Objects can communicate with each
other over the network via Session objects, and only need to work with the
dict-based IPython message spec. The Session will handle
serialization/deserialization, security, and metadata.
Sessions support configurable serialization via packer/unpacker traits,
and signing with HMAC digests via the key/keyfile traits.
Parameters
----------
debug : bool
whether to trigger extra debugging statements
packer/unpacker : str : 'json', 'pickle' or import_string
importstrings for methods to serialize message parts. If just
'json' or 'pickle', predefined JSON and pickle packers will be used.
Otherwise, the entire importstring must be used.
The functions must accept at least valid JSON input, and output *bytes*.
For example, to use msgpack:
packer = 'msgpack.packb', unpacker='msgpack.unpackb'
pack/unpack : callables
You can also set the pack/unpack callables for serialization directly.
session : bytes
the ID of this Session object. The default is to generate a new UUID.
username : unicode
username added to message headers. The default is to ask the OS.
key : bytes
The key used to initialize an HMAC signature. If unset, messages
will not be signed or checked.
keyfile : filepath
The file containing a key. If this is set, `key` will be initialized
to the contents of the file.
"""
debug = Bool(False, config=True, help="""Debug output in the Session""")
check_pid = Bool(True, config=True,
help="""Whether to check PID to protect against calls after fork.
This check can be disabled if fork-safety is handled elsewhere.
""")
packer = DottedObjectName('json',config=True,
help="""The name of the packer for serializing messages.
Should be one of 'json', 'pickle', or an import name
for a custom callable serializer.""")
@observe('packer')
def _packer_changed(self, change):
new = change['new']
if new.lower() == 'json':
self.pack = json_packer
self.unpack = json_unpacker
self.unpacker = new
elif new.lower() == 'pickle':
self.pack = pickle_packer
self.unpack = pickle_unpacker
self.unpacker = new
else:
self.pack = import_item(str(new))
unpacker = DottedObjectName('json', config=True,
help="""The name of the unpacker for unserializing messages.
Only used with custom functions for `packer`.""")
@observe('unpacker')
def _unpacker_changed(self, change):
new = change['new']
if new.lower() == 'json':
self.pack = json_packer
self.unpack = json_unpacker
self.packer = new
elif new.lower() == 'pickle':
self.pack = pickle_packer
self.unpack = pickle_unpacker
self.packer = new
else:
self.unpack = import_item(str(new))
session = CUnicode(u'', config=True,
help="""The UUID identifying this session.""")
def _session_default(self):
u = new_id()
self.bsession = u.encode('ascii')
return u
@observe('session')
def _session_changed(self, change):
self.bsession = self.session.encode('ascii')
# bsession is the session as bytes
bsession = CBytes(b'')
username = Unicode(str_to_unicode(os.environ.get('USER', 'username')),
help="""Username for the Session. Default is your system username.""",
config=True)
metadata = Dict({}, config=True,
help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""")
# if 0, no adapting to do.
adapt_version = Integer(0)
# message signature related traits:
key = CBytes(config=True,
help="""execution key, for signing messages.""")
def _key_default(self):
return new_id_bytes()
@observe('key')
def _key_changed(self, change):
self._new_auth()
signature_scheme = Unicode('hmac-sha256', config=True,
help="""The digest scheme used to construct the message signatures.
Must have the form 'hmac-HASH'.""")
@observe('signature_scheme')
def _signature_scheme_changed(self, change):
new = change['new']
if not new.startswith('hmac-'):
raise TraitError("signature_scheme must start with 'hmac-', got %r" % new)
hash_name = new.split('-', 1)[1]
try:
self.digest_mod = getattr(hashlib, hash_name)
except AttributeError:
raise TraitError("hashlib has no such attribute: %s" % hash_name)
self._new_auth()
digest_mod = Any()
def _digest_mod_default(self):
return hashlib.sha256
auth = Instance(hmac.HMAC, allow_none=True)
def _new_auth(self):
if self.key:
self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod)
else:
self.auth = None
digest_history = Set()
digest_history_size = Integer(2**16, config=True,
help="""The maximum number of digests to remember.
The digest history will be culled when it exceeds this value.
"""
)
keyfile = Unicode('', config=True,
help="""path to file containing execution key.""")
@observe('keyfile')
def _keyfile_changed(self, change):
with open(change['new'], 'rb') as f:
self.key = f.read().strip()
# for protecting against sends from forks
pid = Integer()
# serialization traits:
pack = Any(default_packer) # the actual packer function
@observe('pack')
def _pack_changed(self, change):
new = change['new']
if not callable(new):
raise TypeError("packer must be callable, not %s"%type(new))
unpack = Any(default_unpacker) # the actual packer function
@observe('unpack')
def _unpack_changed(self, change):
# unpacker is not checked - it is assumed to be
new = change['new']
if not callable(new):
raise TypeError("unpacker must be callable, not %s"%type(new))
# thresholds:
copy_threshold = Integer(2**16, config=True,
help="Threshold (in bytes) beyond which a buffer should be sent without copying.")
buffer_threshold = Integer(MAX_BYTES, config=True,
help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.")
item_threshold = Integer(MAX_ITEMS, config=True,
help="""The maximum number of items for a container to be introspected for custom serialization.
Containers larger than this are pickled outright.
"""
)
def __init__(self, **kwargs):
"""create a Session object
Parameters
----------
debug : bool
whether to trigger extra debugging statements
packer/unpacker : str : 'json', 'pickle' or import_string
importstrings for methods to serialize message parts. If just
'json' or 'pickle', predefined JSON and pickle packers will be used.
Otherwise, the entire importstring must be used.
The functions must accept at least valid JSON input, and output
*bytes*.
For example, to use msgpack:
packer = 'msgpack.packb', unpacker='msgpack.unpackb'
pack/unpack : callables
You can also set the pack/unpack callables for serialization
directly.
session : unicode (must be ascii)
the ID of this Session object. The default is to generate a new
UUID.
bsession : bytes
The session as bytes
username : unicode
username added to message headers. The default is to ask the OS.
key : bytes
The key used to initialize an HMAC signature. If unset, messages
will not be signed or checked.
signature_scheme : str
The message digest scheme. Currently must be of the form 'hmac-HASH',
where 'HASH' is a hashing function available in Python's hashlib.
The default is 'hmac-sha256'.
This is ignored if 'key' is empty.
keyfile : filepath
The file containing a key. If this is set, `key` will be
initialized to the contents of the file.
"""
super(Session, self).__init__(**kwargs)
self._check_packers()
self.none = self.pack({})
# ensure self._session_default() if necessary, so bsession is defined:
self.session
self.pid = os.getpid()
self._new_auth()
if not self.key:
get_logger().warning("Message signing is disabled. This is insecure and not recommended!")
def clone(self):
"""Create a copy of this Session
Useful when connecting multiple times to a given kernel.
This prevents a shared digest_history warning about duplicate digests
due to multiple connections to IOPub in the same process.
.. versionadded:: 5.1
"""
# make a copy
new_session = type(self)()
for name in self.traits():
setattr(new_session, name, getattr(self, name))
# fork digest_history
new_session.digest_history = set()
new_session.digest_history.update(self.digest_history)
return new_session
@property
def msg_id(self):
"""always return new uuid"""
return new_id()
def _check_packers(self):
"""check packers for datetime support."""
pack = self.pack
unpack = self.unpack
# check simple serialization
msg = dict(a=[1,'hi'])
try:
packed = pack(msg)
except Exception as e:
msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}"
if self.packer == 'json':
jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
else:
jsonmsg = ""
raise ValueError(
msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg)
)
# ensure packed message is bytes
if not isinstance(packed, bytes):
raise ValueError("message packed to %r, but bytes are required"%type(packed))
# check that unpack is pack's inverse
try:
unpacked = unpack(packed)
assert unpacked == msg
except Exception as e:
msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}"
if self.packer == 'json':
jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
else:
jsonmsg = ""
raise ValueError(
msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg)
)
# check datetime support
msg = dict(t=utcnow())
try:
unpacked = unpack(pack(msg))
if isinstance(unpacked['t'], datetime):
raise ValueError("Shouldn't deserialize to datetime")
except Exception:
self.pack = lambda o: pack(squash_dates(o))
self.unpack = lambda s: unpack(s)
def msg_header(self, msg_type):
return msg_header(self.msg_id, msg_type, self.username, self.session)
def msg(self, msg_type, content=None, parent=None, header=None, metadata=None):
"""Return the nested message dict.
This format is different from what is sent over the wire. The
serialize/deserialize methods converts this nested message dict to the wire
format, which is a list of message parts.
"""
msg = {}
header = self.msg_header(msg_type) if header is None else header
msg['header'] = header
msg['msg_id'] = header['msg_id']
msg['msg_type'] = header['msg_type']
msg['parent_header'] = {} if parent is None else extract_header(parent)
msg['content'] = {} if content is None else content
msg['metadata'] = self.metadata.copy()
if metadata is not None:
msg['metadata'].update(metadata)
return msg
def sign(self, msg_list):
"""Sign a message with HMAC digest. If no auth, return b''.
Parameters
----------
msg_list : list
The [p_header,p_parent,p_content] part of the message list.
"""
if self.auth is None:
return b''
h = self.auth.copy()
for m in msg_list:
h.update(m)
return str_to_bytes(h.hexdigest())
def serialize(self, msg, ident=None):
"""Serialize the message components to bytes.
This is roughly the inverse of deserialize. The serialize/deserialize
methods work with full message lists, whereas pack/unpack work with
the individual message parts in the message list.
Parameters
----------
msg : dict or Message
The next message dict as returned by the self.msg method.
Returns
-------
msg_list : list
The list of bytes objects to be sent with the format::
[ident1, ident2, ..., DELIM, HMAC, p_header, p_parent,
p_metadata, p_content, buffer1, buffer2, ...]
In this list, the ``p_*`` entities are the packed or serialized
versions, so if JSON is used, these are utf8 encoded JSON strings.
"""
content = msg.get('content', {})
if content is None:
content = self.none
elif isinstance(content, dict):
content = self.pack(content)
elif isinstance(content, bytes):
# content is already packed, as in a relayed message
pass
elif isinstance(content, unicode_type):
# should be bytes, but JSON often spits out unicode
content = content.encode('utf8')
else:
raise TypeError("Content incorrect type: %s"%type(content))
real_message = [self.pack(msg['header']),
self.pack(msg['parent_header']),
self.pack(msg['metadata']),
content,
]
to_send = []
if isinstance(ident, list):
# accept list of idents
to_send.extend(ident)
elif ident is not None:
to_send.append(ident)
to_send.append(DELIM)
signature = self.sign(real_message)
to_send.append(signature)
to_send.extend(real_message)
return to_send
def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
buffers=None, track=False, header=None, metadata=None):
"""Build and send a message via stream or socket.
The message format used by this function internally is as follows:
[ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
buffer1,buffer2,...]
The serialize/deserialize methods convert the nested message dict into this
format.
Parameters
----------
stream : zmq.Socket or ZMQStream
The socket-like object used to send the data.
msg_or_type : str or Message/dict
Normally, msg_or_type will be a msg_type unless a message is being
sent more than once. If a header is supplied, this can be set to
None and the msg_type will be pulled from the header.
content : dict or None
The content of the message (ignored if msg_or_type is a message).
header : dict or None
The header dict for the message (ignored if msg_to_type is a message).
parent : Message or dict or None
The parent or parent header describing the parent of this message
(ignored if msg_or_type is a message).
ident : bytes or list of bytes
The zmq.IDENTITY routing path.
metadata : dict or None
The metadata describing the message
buffers : list or None
The already-serialized buffers to be appended to the message.
track : bool
Whether to track. Only for use with Sockets, because ZMQStream
objects cannot track messages.
Returns
-------
msg : dict
The constructed message.
"""
if not isinstance(stream, zmq.Socket):
# ZMQStreams and dummy sockets do not support tracking.
track = False
if isinstance(msg_or_type, (Message, dict)):
# We got a Message or message dict, not a msg_type so don't
# build a new Message.
msg = msg_or_type
buffers = buffers or msg.get('buffers', [])
else:
msg = self.msg(msg_or_type, content=content, parent=parent,
header=header, metadata=metadata)
if self.check_pid and not os.getpid() == self.pid:
get_logger().warning("WARNING: attempted to send message from fork\n%s",
msg
)
return
buffers = [] if buffers is None else buffers
for idx, buf in enumerate(buffers):
if isinstance(buf, memoryview):
view = buf
else:
try:
# check to see if buf supports the buffer protocol.
view = memoryview(buf)
except TypeError:
raise TypeError("Buffer objects must support the buffer protocol.")
# memoryview.contiguous is new in 3.3,
# just skip the check on Python 2
if hasattr(view, 'contiguous') and not view.contiguous:
# zmq requires memoryviews to be contiguous
raise ValueError("Buffer %i (%r) is not contiguous" % (idx, buf))
if self.adapt_version:
msg = adapt(msg, self.adapt_version)
to_send = self.serialize(msg, ident)
to_send.extend(buffers)
longest = max([ len(s) for s in to_send ])
copy = (longest < self.copy_threshold)
if buffers and track and not copy:
# only really track when we are doing zero-copy buffers
tracker = stream.send_multipart(to_send, copy=False, track=True)
else:
# use dummy tracker, which will be done immediately
tracker = DONE
stream.send_multipart(to_send, copy=copy)
if self.debug:
pprint.pprint(msg)
pprint.pprint(to_send)
pprint.pprint(buffers)
msg['tracker'] = tracker
return msg
def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
"""Send a raw message via ident path.
This method is used to send a already serialized message.
Parameters
----------
stream : ZMQStream or Socket
The ZMQ stream or socket to use for sending the message.
msg_list : list
The serialized list of messages to send. This only includes the
[p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of
the message.
ident : ident or list
A single ident or a list of idents to use in sending.
"""
to_send = []
if isinstance(ident, bytes):
ident = [ident]
if ident is not None:
to_send.extend(ident)
to_send.append(DELIM)
# Don't include buffers in signature (per spec).
to_send.append(self.sign(msg_list[0:4]))
to_send.extend(msg_list)
stream.send_multipart(to_send, flags, copy=copy)
def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
"""Receive and unpack a message.
Parameters
----------
socket : ZMQStream or Socket
The socket or stream to use in receiving.
Returns
-------
[idents], msg
[idents] is a list of idents and msg is a nested message dict of
same format as self.msg returns.
"""
if isinstance(socket, ZMQStream):
socket = socket.socket
try:
msg_list = socket.recv_multipart(mode, copy=copy)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
# We can convert EAGAIN to None as we know in this case
# recv_multipart won't return None.
return None,None
else:
raise
# split multipart message into identity list and message dict
# invalid large messages can cause very expensive string comparisons
idents, msg_list = self.feed_identities(msg_list, copy)
try:
return idents, self.deserialize(msg_list, content=content, copy=copy)
except Exception as e:
# TODO: handle it
raise e
def feed_identities(self, msg_list, copy=True):
"""Split the identities from the rest of the message.
Feed until DELIM is reached, then return the prefix as idents and
remainder as msg_list. This is easily broken by setting an IDENT to DELIM,
but that would be silly.
Parameters
----------
msg_list : a list of Message or bytes objects
The message to be split.
copy : bool
flag determining whether the arguments are bytes or Messages
Returns
-------
(idents, msg_list) : two lists
idents will always be a list of bytes, each of which is a ZMQ
identity. msg_list will be a list of bytes or zmq.Messages of the
form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and
should be unpackable/unserializable via self.deserialize at this
point.
"""
if copy:
idx = msg_list.index(DELIM)
return msg_list[:idx], msg_list[idx+1:]
else:
failed = True
for idx,m in enumerate(msg_list):
if m.bytes == DELIM:
failed = False
break
if failed:
raise ValueError("DELIM not in msg_list")
idents, msg_list = msg_list[:idx], msg_list[idx+1:]
return [m.bytes for m in idents], msg_list
def _add_digest(self, signature):
"""add a digest to history to protect against replay attacks"""
if self.digest_history_size == 0:
# no history, never add digests
return
self.digest_history.add(signature)
if len(self.digest_history) > self.digest_history_size:
# threshold reached, cull 10%
self._cull_digest_history()
def _cull_digest_history(self):
"""cull the digest history
Removes a randomly selected 10% of the digest history
"""
current = len(self.digest_history)
n_to_cull = max(int(current // 10), current - self.digest_history_size)
if n_to_cull >= current:
self.digest_history = set()
return
to_cull = random.sample(self.digest_history, n_to_cull)
self.digest_history.difference_update(to_cull)
def deserialize(self, msg_list, content=True, copy=True):
"""Unserialize a msg_list to a nested message dict.
This is roughly the inverse of serialize. The serialize/deserialize
methods work with full message lists, whereas pack/unpack work with
the individual message parts in the message list.
Parameters
----------
msg_list : list of bytes or Message objects
The list of message parts of the form [HMAC,p_header,p_parent,
p_metadata,p_content,buffer1,buffer2,...].
content : bool (True)
Whether to unpack the content dict (True), or leave it packed
(False).
copy : bool (True)
Whether msg_list contains bytes (True) or the non-copying Message
objects in each place (False).
Returns
-------
msg : dict
The nested message dict with top-level keys [header, parent_header,
content, buffers]. The buffers are returned as memoryviews.
"""
minlen = 5
message = {}
if not copy:
# pyzmq didn't copy the first parts of the message, so we'll do it
for i in range(minlen):
msg_list[i] = msg_list[i].bytes
if self.auth is not None:
signature = msg_list[0]
if not signature:
raise ValueError("Unsigned Message")
if signature in self.digest_history:
raise ValueError("Duplicate Signature: %r" % signature)
if content:
# Only store signature if we are unpacking content, don't store if just peeking.
self._add_digest(signature)
check = self.sign(msg_list[1:5])
if not compare_digest(signature, check):
raise ValueError("Invalid Signature: %r" % signature)
if not len(msg_list) >= minlen:
raise TypeError("malformed message, must have at least %i elements"%minlen)
header = self.unpack(msg_list[1])
message['header'] = extract_dates(header)
message['msg_id'] = header['msg_id']
message['msg_type'] = header['msg_type']
message['parent_header'] = extract_dates(self.unpack(msg_list[2]))
message['metadata'] = self.unpack(msg_list[3])
if content:
message['content'] = self.unpack(msg_list[4])
else:
message['content'] = msg_list[4]
buffers = [memoryview(b) for b in msg_list[5:]]
if buffers and buffers[0].shape is None:
# force copy to workaround pyzmq #646
buffers = [memoryview(b.bytes) for b in msg_list[5:]]
message['buffers'] = buffers
if self.debug:
pprint.pprint(message)
# adapt to the current version
return adapt(message)
def unserialize(self, *args, **kwargs):
warnings.warn(
"Session.unserialize is deprecated. Use Session.deserialize.",
DeprecationWarning,
)
return self.deserialize(*args, **kwargs)
class GitLabOAuthenticator(OAuthenticator):
# see gitlab_scopes.md for details about scope config
# set scopes via config, e.g.
# c.GitLabOAuthenticator.scope = ['read_user']
_deprecated_oauth_aliases = {
"gitlab_group_whitelist": ("allowed_gitlab_groups", "0.12.0"),
"gitlab_project_id_whitelist": ("allowed_project_ids", "0.12.0"),
**OAuthenticator._deprecated_oauth_aliases,
}
login_service = "GitLab"
client_id_env = 'GITLAB_CLIENT_ID'
client_secret_env = 'GITLAB_CLIENT_SECRET'
gitlab_url = Unicode("https://gitlab.com", config=True)
@default("gitlab_url")
def _default_gitlab_url(self):
"""get default gitlab url from env"""
gitlab_url = os.getenv('GITLAB_URL')
gitlab_host = os.getenv('GITLAB_HOST')
if not gitlab_url and gitlab_host:
warnings.warn(
'Use of GITLAB_HOST might be deprecated in the future. '
'Rename GITLAB_HOST environment variable to GITLAB_URL.',
PendingDeprecationWarning,
)
if gitlab_host.startswith(('https:', 'http:')):
gitlab_url = gitlab_host
else:
# Hides common mistake of users which set the GITLAB_HOST
# without a protocol specification.
gitlab_url = 'https://{0}'.format(gitlab_host)
warnings.warn(
'The https:// prefix has been added to GITLAB_HOST.'
'Set GITLAB_URL="{0}" instead.'.format(gitlab_host))
# default to gitlab.com
if not gitlab_url:
gitlab_url = 'https://gitlab.com'
return gitlab_url
gitlab_api_version = CUnicode('4', config=True)
@default('gitlab_api_version')
def _gitlab_api_version_default(self):
return os.environ.get('GITLAB_API_VERSION') or '4'
gitlab_api = Unicode(config=True)
@default("gitlab_api")
def _default_gitlab_api(self):
return '%s/api/v%s' % (self.gitlab_url, self.gitlab_api_version)
@default("authorize_url")
def _authorize_url_default(self):
return "%s/oauth/authorize" % self.gitlab_url
@default("token_url")
def _token_url_default(self):
return "%s/oauth/access_token" % self.gitlab_url
gitlab_group_whitelist = Set(
help="Deprecated, use `GitLabOAuthenticator.allowed_gitlab_groups`",
config=True,
)
allowed_gitlab_groups = Set(
config=True, help="Automatically allow members of selected groups")
gitlab_project_id_whitelist = Set(
help="Deprecated, use `GitLabOAuthenticator.allowed_project_ids`",
config=True,
)
allowed_project_ids = Set(
config=True,
help=
"Automatically allow members with Developer access to selected project ids",
)
gitlab_version = None
async def authenticate(self, handler, data=None):
code = handler.get_argument("code")
# Exchange the OAuth code for a GitLab Access Token
#
# See: https://github.com/gitlabhq/gitlabhq/blob/master/doc/api/oauth2.md
# GitLab specifies a POST request yet requires URL parameters
params = dict(
client_id=self.client_id,
client_secret=self.client_secret,
code=code,
grant_type="authorization_code",
redirect_uri=self.get_callback_url(handler),
)
validate_server_cert = self.validate_server_cert
url = url_concat("%s/oauth/token" % self.gitlab_url, params)
req = HTTPRequest(
url,
method="POST",
headers={"Accept": "application/json"},
validate_cert=validate_server_cert,
body='', # Body is required for a POST...
)
resp_json = await self.fetch(req, label="getting access token")
access_token = resp_json['access_token']
# memoize gitlab version for class lifetime
if self.gitlab_version is None:
self.gitlab_version = await self._get_gitlab_version(access_token)
self.member_api_variant = 'all/' if self.gitlab_version >= [
12, 4
] else ''
# Determine who the logged in user is
req = HTTPRequest(
"%s/user" % self.gitlab_api,
method="GET",
validate_cert=validate_server_cert,
headers=_api_headers(access_token),
)
resp_json = await self.fetch(req, label="getting gitlab user")
username = resp_json["username"]
user_id = resp_json["id"]
is_admin = resp_json.get("is_admin", False)
# Check if user is a member of any allowed groups or projects.
# These checks are performed here, as it requires `access_token`.
user_in_group = user_in_project = False
is_group_specified = is_project_id_specified = False
if self.allowed_gitlab_groups:
is_group_specified = True
user_in_group = await self._check_membership_allowed_groups(
user_id, access_token)
# We skip project_id check if user is in allowed group.
if self.allowed_project_ids and not user_in_group:
is_project_id_specified = True
user_in_project = await self._check_membership_allowed_project_ids(
user_id, access_token)
no_config_specified = not (is_group_specified
or is_project_id_specified)
if ((is_group_specified and user_in_group)
or (is_project_id_specified and user_in_project)
or no_config_specified):
return {
'name': username,
'auth_state': {
'access_token': access_token,
'gitlab_user': resp_json
},
}
else:
self.log.warning("%s not in group or project allowed list",
username)
return None
async def _get_gitlab_version(self, access_token):
url = '%s/version' % self.gitlab_api
req = HTTPRequest(
url,
method="GET",
headers=_api_headers(access_token),
validate_cert=self.validate_server_cert,
)
resp_json = await self.fetch(req)
version_strings = resp_json['version'].split('-')[0].split('.')[:3]
version_ints = list(map(int, version_strings))
return version_ints
async def _check_membership_allowed_groups(self, user_id, access_token):
headers = _api_headers(access_token)
# Check if user is a member of any group in the allowed list
for group in map(url_escape, self.allowed_gitlab_groups):
url = "%s/groups/%s/members/%s%d" % (
self.gitlab_api,
quote(group, safe=''),
self.member_api_variant,
user_id,
)
req = HTTPRequest(
url,
method="GET",
headers=headers,
validate_cert=self.validate_server_cert,
)
resp = await self.fetch(req, raise_error=False, parse_json=False)
if resp.code == 200:
return True # user _is_ in group
return False
async def _check_membership_allowed_project_ids(self, user_id,
access_token):
headers = _api_headers(access_token)
# Check if user has developer access to any project in the allowed list
for project in self.allowed_project_ids:
url = "%s/projects/%s/members/%s%d" % (
self.gitlab_api,
project,
self.member_api_variant,
user_id,
)
req = HTTPRequest(
url,
method="GET",
headers=headers,
validate_cert=self.validate_server_cert,
)
resp_json = await self.fetch(req, raise_error=False)
if resp_json:
access_level = resp_json.get('access_level', 0)
# We only allow access level Developer and above
# Reference: https://docs.gitlab.com/ee/api/members.html
if access_level >= 30:
return True
return False
class JupyterConsoleApp(ConnectionFileMixin):
name = 'jupyter-console-mixin'
description = """
The Jupyter Console Mixin.
This class contains the common portions of console client (QtConsole,
ZMQ-based terminal console, etc). It is not a full console, in that
launched terminal subprocesses will not be able to accept input.
The Console using this mixing supports various extra features beyond
the single-process Terminal IPython shell, such as connecting to
existing kernel, via:
jupyter console <appname> --existing
as well as tunnel via SSH
"""
classes = classes
flags = Dict(flags)
aliases = Dict(aliases)
kernel_manager_class = Type(default_value=KernelManager,
config=True,
help='The kernel manager class to use.')
kernel_client_class = BlockingKernelClient
kernel_argv = List(Unicode())
# connection info:
sshserver = Unicode(
'',
config=True,
help="""The SSH server to use to connect to the kernel.""")
sshkey = Unicode(
'',
config=True,
help="""Path to the ssh key to use for logging in to the ssh server."""
)
def _connection_file_default(self):
return 'kernel-%i.json' % os.getpid()
existing = CUnicode('',
config=True,
help="""Connect to an already running kernel""")
kernel_name = Unicode('python',
config=True,
help="""The name of the default kernel to start.""")
confirm_exit = CBool(
True,
config=True,
help="""
Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
to force a direct exit without any confirmation.""",
)
def build_kernel_argv(self, argv=None):
"""build argv to be passed to kernel subprocess
Override in subclasses if any args should be passed to the kernel
"""
self.kernel_argv = self.extra_args
def init_connection_file(self):
"""find the connection file, and load the info if found.
The current working directory and the current profile's security
directory will be searched for the file if it is not given by
absolute path.
When attempting to connect to an existing kernel and the `--existing`
argument does not match an existing file, it will be interpreted as a
fileglob, and the matching file in the current profile's security dir
with the latest access time will be used.
After this method is called, self.connection_file contains the *full path*
to the connection file, never just its name.
"""
if self.existing:
try:
cf = find_connection_file(self.existing,
['.', self.runtime_dir])
except Exception:
self.log.critical(
"Could not find existing kernel connection file %s",
self.existing)
self.exit(1)
self.log.debug("Connecting to existing kernel: %s" % cf)
self.connection_file = cf
else:
# not existing, check if we are going to write the file
# and ensure that self.connection_file is a full path, not just the shortname
try:
cf = find_connection_file(self.connection_file,
[self.runtime_dir])
except Exception:
# file might not exist
if self.connection_file == os.path.basename(
self.connection_file):
# just shortname, put it in security dir
cf = os.path.join(self.runtime_dir, self.connection_file)
else:
cf = self.connection_file
self.connection_file = cf
try:
self.connection_file = filefind(self.connection_file,
['.', self.runtime_dir])
except IOError:
self.log.debug("Connection File not found: %s",
self.connection_file)
return
# should load_connection_file only be used for existing?
# as it is now, this allows reusing ports if an existing
# file is requested
try:
self.load_connection_file()
except Exception:
self.log.error("Failed to load connection file: %r",
self.connection_file,
exc_info=True)
self.exit(1)
def init_ssh(self):
"""set up ssh tunnels, if needed."""
if not self.existing or (not self.sshserver and not self.sshkey):
return
self.load_connection_file()
transport = self.transport
ip = self.ip
if transport != 'tcp':
self.log.error("Can only use ssh tunnels with TCP sockets, not %s",
transport)
sys.exit(-1)
if self.sshkey and not self.sshserver:
# specifying just the key implies that we are connecting directly
self.sshserver = ip
ip = localhost()
# build connection dict for tunnels:
info = dict(ip=ip,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port)
self.log.info("Forwarding connections to %s via %s" %
(ip, self.sshserver))
# tunnels return a new set of ports, which will be on localhost:
self.ip = localhost()
try:
newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
except:
# even catch KeyboardInterrupt
self.log.error("Could not setup tunnels", exc_info=True)
self.exit(1)
self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
cf = self.connection_file
root, ext = os.path.splitext(cf)
self.connection_file = root + '-ssh' + ext
self.write_connection_file() # write the new connection file
self.log.info("To connect another client via this tunnel, use:")
self.log.info("--existing %s" % os.path.basename(self.connection_file))
def _new_connection_file(self):
cf = ''
while not cf:
# we don't need a 128b id to distinguish kernels, use more readable
# 48b node segment (12 hex chars). Users running more than 32k simultaneous
# kernels can subclass.
ident = str(uuid.uuid4()).split('-')[-1]
cf = os.path.join(self.runtime_dir, 'kernel-%s.json' % ident)
# only keep if it's actually new. Protect against unlikely collision
# in 48b random search space
cf = cf if not os.path.exists(cf) else ''
return cf
def init_kernel_manager(self):
# Don't let Qt or ZMQ swallow KeyboardInterupts.
if self.existing:
self.kernel_manager = None
return
signal.signal(signal.SIGINT, signal.SIG_DFL)
# Create a KernelManager and start a kernel.
try:
self.kernel_manager = self.kernel_manager_class(
ip=self.ip,
session=self.session,
transport=self.transport,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
connection_file=self.connection_file,
kernel_name=self.kernel_name,
parent=self,
data_dir=self.data_dir,
)
except NoSuchKernel:
self.log.critical("Could not find kernel %s", self.kernel_name)
self.exit(1)
self.kernel_manager.client_factory = self.kernel_client_class
# FIXME: remove special treatment of IPython kernels
kwargs = {}
if self.kernel_manager.ipykernel:
kwargs['extra_arguments'] = self.kernel_argv
self.kernel_manager.start_kernel(**kwargs)
atexit.register(self.kernel_manager.cleanup_ipc_files)
if self.sshserver:
# ssh, write new connection file
self.kernel_manager.write_connection_file()
# in case KM defaults / ssh writing changes things:
km = self.kernel_manager
self.shell_port = km.shell_port
self.iopub_port = km.iopub_port
self.stdin_port = km.stdin_port
self.hb_port = km.hb_port
self.connection_file = km.connection_file
atexit.register(self.kernel_manager.cleanup_connection_file)
def init_kernel_client(self):
if self.kernel_manager is not None:
self.kernel_client = self.kernel_manager.client()
else:
self.kernel_client = self.kernel_client_class(
session=self.session,
ip=self.ip,
transport=self.transport,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
connection_file=self.connection_file,
parent=self,
)
self.kernel_client.start_channels()
def initialize(self, argv=None):
"""
Classes which mix this class in should call:
JupyterConsoleApp.initialize(self,argv)
"""
if self._dispatching:
return
self.init_connection_file()
self.init_ssh()
self.init_kernel_manager()
self.init_kernel_client()
class AppLayout(GridBox, LayoutProperties):
""" Define an application like layout of widgets.
Parameters
----------
header: instance of Widget
left_sidebar: instance of Widget
center: instance of Widget
right_sidebar: instance of Widget
footer: instance of Widget
widgets to fill the positions in the layout
merge: bool
flag to say whether the empty positions should be automatically merged
pane_widths: list of numbers/strings
the fraction of the total layout width each of the central panes should occupy
(left_sidebar,
center, right_sidebar)
pane_heights: list of numbers/strings
the fraction of the width the vertical space that the panes should occupy
(left_sidebar, center, right_sidebar)
{style_params}
Examples
--------
"""
# widget positions
header = Instance(Widget, allow_none=True)
footer = Instance(Widget, allow_none=True)
left_sidebar = Instance(Widget, allow_none=True)
right_sidebar = Instance(Widget, allow_none=True)
center = Instance(Widget, allow_none=True)
# extra args
pane_widths = Tuple(CUnicode(),
CUnicode(),
CUnicode(),
default_value=['1fr', '2fr', '1fr'])
pane_heights = Tuple(CUnicode(),
CUnicode(),
CUnicode(),
default_value=['1fr', '3fr', '1fr'])
merge = Bool(default_value=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._update_layout()
@staticmethod
def _size_to_css(size):
if re.match(r'\d+\.?\d*(px|fr|%)$', size):
return size
if re.match(r'\d+\.?\d*$', size):
return size + 'fr'
raise TypeError(
"the pane sizes must be in one of the following formats: "
"'10px', '10fr', 10 (will be converted to '10fr')."
"Got '{}'".format(size))
def _convert_sizes(self, size_list):
return list(map(self._size_to_css, size_list))
def _update_layout(self):
grid_template_areas = [["header", "header", "header"],
["left-sidebar", "center", "right-sidebar"],
["footer", "footer", "footer"]]
grid_template_columns = self._convert_sizes(self.pane_widths)
grid_template_rows = self._convert_sizes(self.pane_heights)
all_children = {
'header': self.header,
'footer': self.footer,
'left-sidebar': self.left_sidebar,
'right-sidebar': self.right_sidebar,
'center': self.center
}
children = {
position: child
for position, child in all_children.items() if child is not None
}
if not children:
return
for position, child in children.items():
child.layout.grid_area = position
if self.merge:
if len(children) == 1:
position = list(children.keys())[0]
grid_template_areas = [[position, position, position],
[position, position, position],
[position, position, position]]
else:
if self.center is None:
for row in grid_template_areas:
del row[1]
del grid_template_columns[1]
if self.left_sidebar is None:
grid_template_areas[1][0] = grid_template_areas[1][1]
if self.right_sidebar is None:
grid_template_areas[1][-1] = grid_template_areas[1][-2]
if (self.left_sidebar is None and self.right_sidebar is None
and self.center is None):
grid_template_areas = [['header'], ['footer']]
grid_template_columns = ['1fr']
grid_template_rows = ['1fr', '1fr']
if self.header is None:
del grid_template_areas[0]
del grid_template_rows[0]
if self.footer is None:
del grid_template_areas[-1]
del grid_template_rows[-1]
grid_template_areas_css = "\n".join('"{}"'.format(" ".join(line))
for line in grid_template_areas)
self.layout.grid_template_columns = " ".join(grid_template_columns)
self.layout.grid_template_rows = " ".join(grid_template_rows)
self.layout.grid_template_areas = grid_template_areas_css
self.children = tuple(children.values())
@observe("footer", "header", "center", "left_sidebar", "right_sidebar",
"merge", "pane_widths", "pane_heights")
def _child_changed(self, change): #pylint: disable=unused-argument
self._update_layout()