def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
if sys.stdout is not None:
sys.stdout.flush()
e_stdout = None if self.quiet else sys.__stdout__
e_stderr = None if self.quiet else sys.__stderr__
sys.stdout = outstream_factory(self.session,
self.iopub_thread,
'stdout',
echo=e_stdout)
if sys.stderr is not None:
sys.stderr.flush()
sys.stderr = outstream_factory(self.session,
self.iopub_thread,
'stderr',
echo=e_stderr)
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
self.displayhook = displayhook_factory(self.session,
self.iopub_socket)
sys.displayhook = self.displayhook
self.patch_io()
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
sys.stdout = outstream_factory(self.session, self.iopub_socket,
u'stdout')
sys.stderr = outstream_factory(self.session, self.iopub_socket,
u'stderr')
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
sys.displayhook = displayhook_factory(self.session,
self.iopub_socket)
def get(self, bundler_id):
'''
Executes the requested bundler on the given notebook.
:param bundler_id: Unique ID of an installed bundler
:arg notebook: Path to the notebook relative to the notebook directory
root
'''
notebook = self.get_query_argument('notebook')
abs_nb_path = os.path.join(self.notebook_dir, url2path(notebook))
try:
bundler = self.get_bundler(bundler_id)
except KeyError:
raise web.HTTPError(404, 'Bundler %s not found' % bundler_id)
module_name = bundler['module_name']
try:
# no-op in python3, decode error in python2
module_name = str(module_name)
except UnicodeEncodeError:
# Encode unicode as utf-8 in python2 else import_item fails
module_name = module_name.encode('utf-8')
try:
bundler_mod = import_item(module_name)
except ImportError:
raise web.HTTPError(500,
'Could not import bundler %s ' % bundler_id)
# Let the bundler respond in any way it sees fit and assume it will
# finish the request
yield gen.maybe_future(bundler_mod.bundle(self, abs_nb_path))
def get(self, path):
"""Bundle the given notebook.
Parameters
----------
path: str
Path to the notebook (path parameter)
bundler: str
Bundler ID to use (query parameter)
"""
bundler_id = self.get_query_argument('bundler')
model = self.contents_manager.get(path=url2path(path))
try:
bundler = self.get_bundler(bundler_id)
except KeyError:
raise web.HTTPError(400, 'Bundler %s not enabled' % bundler_id)
module_name = bundler['module_name']
try:
# no-op in python3, decode error in python2
module_name = str(module_name)
except UnicodeEncodeError:
# Encode unicode as utf-8 in python2 else import_item fails
module_name = module_name.encode('utf-8')
try:
bundler_mod = import_item(module_name)
except ImportError:
raise web.HTTPError(500,
'Could not import bundler %s ' % bundler_id)
# Let the bundler respond in any way it sees fit and assume it will
# finish the request
yield gen.maybe_future(bundler_mod.bundle(self, model))
def get_validator(version=None, version_minor=None):
"""Load the JSON schema into a Validator"""
if version is None:
from .. import current_nbformat
version = current_nbformat
v = import_item("nbformat.v%s" % version)
current_minor = v.nbformat_minor
if version_minor is None:
version_minor = current_minor
version_tuple = (version, version_minor)
if version_tuple not in validators:
try:
v.nbformat_schema
except AttributeError:
# no validator
return None
schema_path = os.path.join(os.path.dirname(v.__file__),
v.nbformat_schema)
with open(schema_path) as f:
schema_json = json.load(f)
if current_minor < version_minor:
# notebook from the future, relax all `additionalProperties: False` requirements
schema_json = _relax_additional_properties(schema_json)
# and allow undefined cell types and outputs
schema_json = _allow_undefined(schema_json)
validators[version_tuple] = Validator(schema_json)
return validators[version_tuple]
def get(self, bundler_id):
"""
Executes the requested bundler on the given notebook.
:param bundler_id: Unique ID of an installed bundler
:arg notebook: Path to the notebook relative to the notebook directory
root
"""
notebook = self.get_query_argument("notebook")
abs_nb_path = os.path.join(self.notebook_dir, url2path(notebook))
try:
bundler = self.get_bundler(bundler_id)
except KeyError:
raise web.HTTPError(404, "Bundler %s not found" % bundler_id)
module_name = bundler["module_name"]
try:
# no-op in python3, decode error in python2
module_name = str(module_name)
except UnicodeEncodeError:
# Encode unicode as utf-8 in python2 else import_item fails
module_name = module_name.encode("utf-8")
try:
bundler_mod = import_item(module_name)
except ImportError:
raise web.HTTPError(500, "Could not import bundler %s " % bundler_id)
# Let the bundler respond in any way it sees fit and assume it will
# finish the request
yield gen.maybe_future(bundler_mod.bundle(self, abs_nb_path))
def get_exporter(name):
""" given an exporter name, return a class ready to be instantiate
Raises ValueError if exporter is not found
"""
if name.lower() in exporter_map:
return exporter_map[name.lower()]
if '.' in name:
try:
return import_item(name)
except ImportError:
log = logging.getLogger()
log.error("Error importing %s" % name, exc_info=True)
pass
else:
try:
return entrypoints.get_single('nbconvert.exporter', name).load()
except entrypoints.NoSuchEntryPoint:
pass
valid_names = sorted(
get_export_names() +
list(entrypoints.get_group_named('nbconvert.exporter')))
raise ValueError('Unknown exporter "%s", did you mean one of: %s?' %
(name, ', '.join(valid_names)))
def get(self, path):
"""Bundle the given notebook.
Parameters
----------
path: str
Path to the notebook (path parameter)
bundler: str
Bundler ID to use (query parameter)
"""
bundler_id = self.get_query_argument('bundler')
model = self.contents_manager.get(path=url2path(path))
try:
bundler = self.get_bundler(bundler_id)
except KeyError:
raise web.HTTPError(400, 'Bundler %s not enabled' % bundler_id)
module_name = bundler['module_name']
try:
# no-op in python3, decode error in python2
module_name = str(module_name)
except UnicodeEncodeError:
# Encode unicode as utf-8 in python2 else import_item fails
module_name = module_name.encode('utf-8')
try:
bundler_mod = import_item(module_name)
except ImportError:
raise web.HTTPError(500, 'Could not import bundler %s ' % bundler_id)
# Let the bundler respond in any way it sees fit and assume it will
# finish the request
yield gen.maybe_future(bundler_mod.bundle(self, model))
def start_kernel(self):
"""Starts a kernel in a separate process.
Where the started kernel resides depends on the configured lifecycle manager.
Parameters
----------
`**kwargs` : optional
keyword arguments that are passed down to build the kernel_cmd
and launching the kernel (e.g. Popen kwargs).
"""
lifecycle_manager_class_name = self.lifecycle_info.get('class_name')
self.log.debug(
"Instantiating kernel '{}' with lifecycle manager: {}".format(
self.kernel_spec.display_name, lifecycle_manager_class_name))
lifecycle_manager_class = import_item(lifecycle_manager_class_name)
self.lifecycle_manager = lifecycle_manager_class(
kernel_manager=self,
lifecycle_config=self.lifecycle_info.get('config', {}))
# format command
kernel_cmd = self.format_kernel_cmd()
self.log.debug("Launching kernel: {} with command: {}".format(
self.kernel_spec.display_name, kernel_cmd))
self.kernel = self.lifecycle_manager.launch_process(kernel_cmd,
env=self.env)
def _validate_pre_save_hook(self, proposal):
value = proposal["value"]
if isinstance(value, str):
value = import_item(self.pre_save_hook)
if not callable(value):
raise TraitError("pre_save_hook must be callable")
return value
def get_validator(version=None, version_minor=None):
"""Load the JSON schema into a Validator"""
if version is None:
from .. import current_nbformat
version = current_nbformat
v = import_item("jupyter_nbformat.v%s" % version)
current_minor = v.nbformat_minor
if version_minor is None:
version_minor = current_minor
version_tuple = (version, version_minor)
if version_tuple not in validators:
try:
v.nbformat_schema
except AttributeError:
# no validator
return None
schema_path = os.path.join(os.path.dirname(v.__file__), v.nbformat_schema)
with open(schema_path) as f:
schema_json = json.load(f)
if current_minor < version_minor:
# notebook from the future, relax all `additionalProperties: False` requirements
schema_json = _relax_additional_properties(schema_json)
# and allow undefined cell types and outputs
schema_json = _allow_undefined(schema_json)
validators[version_tuple] = Validator(schema_json)
return validators[version_tuple]
def get_exporter(name):
""" given an exporter name, return a class ready to be instantiate
Raises ValueError if exporter is not found
"""
if name.lower() in exporter_map:
return exporter_map[name.lower()]
if '.' in name:
try:
return import_item(name)
except ImportError:
log = logging.getLogger()
log.error("Error importing %s" % name, exc_info=True)
pass
else:
try:
return entrypoints.get_single('nbconvert.exporter', name).load()
except entrypoints.NoSuchEntryPoint:
pass
valid_names = sorted(get_export_names() +
list(entrypoints.get_group_named('nbconvert.exporter')))
raise ValueError('Unknown exporter "%s", did you mean one of: %s?'
% (name, ', '.join(valid_names)))
def from_dict(cls: Type[M], schemaspace: str, metadata_dict: dict) -> M:
"""Creates an appropriate instance of Metadata from a dictionary instance"""
# Get the schema and look for metadata_class entry and use that, else Metadata.
metadata_class_name = "elyra.metadata.metadata.Metadata"
schema_name = metadata_dict.get("schema_name")
if schema_name:
schema = SchemaManager.instance().get_schema(
schemaspace, schema_name)
metadata_class_name = schema.get("metadata_class_name",
metadata_class_name)
metadata_class = import_item(metadata_class_name)
try:
instance = metadata_class(**metadata_dict)
if not isinstance(instance, Metadata):
raise ValueError(
f"The metadata_class_name ('{metadata_class_name}') for "
f"schema '{schema_name}' must be a subclass of '{cls.__name__}'!"
)
except TypeError as te:
raise ValueError(
f"The metadata_class_name ('{metadata_class_name}') for "
f"schema '{schema_name}' must be a subclass of '{cls.__name__}'!"
) from te
return instance
def from_dict(cls: Type[M], namespace: str, metadata_dict: dict) -> M:
"""Creates an appropriate instance of Metadata from a dictionary instance """
# Get the schema and look for metadata_class entry and use that, else Metadata.
metadata_class_name = 'elyra.metadata.Metadata'
schema_name = metadata_dict.get('schema_name')
if schema_name:
try:
schema = SchemaManager.instance().get_schema(
namespace, schema_name)
metadata_class_name = schema.get('metadata_class_name',
metadata_class_name)
except Exception: # just use the default
pass
metadata_class = import_item(metadata_class_name)
try:
instance = metadata_class(**metadata_dict)
if not isinstance(instance, Metadata):
raise ValueError(
"The metadata_class_name ('{}') for schema '{}' must be a subclass of '{}'!"
.format(metadata_class_name, schema_name, cls.__name__))
except TypeError as te:
raise ValueError(
"The metadata_class_name ('{}') for schema '{}' must be a subclass of '{}'!"
.format(metadata_class_name, schema_name,
cls.__name__)) from te
return instance
def _validate_post_save_hook(self, proposal):
value = proposal['value']
if isinstance(value, string_types):
value = import_item(value)
if not callable(value):
raise TraitError("post_save_hook must be callable")
return value
def start_kernel(self, **kw):
if self.kernel_spec.process_proxy_class:
self.log.debug("Instantiating kernel '{}' with process proxy: {}".
format(self.kernel_spec.display_name, self.kernel_spec.process_proxy_class))
process_proxy_class = import_item(self.kernel_spec.process_proxy_class)
self.process_proxy = process_proxy_class(kernel_manager=self, proxy_config=self.kernel_spec.process_proxy_config)
return super(RemoteKernelManager, self).start_kernel(**kw)
def start_kernel(self, **kw):
process_proxy = get_process_proxy_config(self.kernel_spec)
process_proxy_class_name = process_proxy.get('class_name')
self.log.debug("Instantiating kernel '{}' with process proxy: {}".
format(self.kernel_spec.display_name, process_proxy_class_name))
process_proxy_class = import_item(process_proxy_class_name)
self.process_proxy = process_proxy_class(kernel_manager=self, proxy_config=process_proxy.get('config'))
self._capture_user_overrides(**kw)
return super(RemoteKernelManager, self).start_kernel(**kw)
def _unpacker_changed(self, name, old, new):
if new.lower() == 'json':
self.pack = json_packer
self.unpack = json_unpacker
self.packer = new
elif new.lower() == 'pickle':
self.pack = pickle_packer
self.unpack = pickle_unpacker
self.packer = new
else:
self.unpack = import_item(str(new))
def _unpacker_changed(self, name, old, new):
if new.lower() == 'json':
self.pack = json_packer
self.unpack = json_unpacker
self.packer = new
elif new.lower() == 'pickle':
self.pack = pickle_packer
self.unpack = pickle_unpacker
self.packer = new
else:
self.unpack = import_item(str(new))
def _unpacker_changed(self, change):
new = change['new']
if new.lower() == 'json':
self.pack = json_packer
self.unpack = json_unpacker
self.packer = new
elif new.lower() == 'pickle':
self.pack = pickle_packer
self.unpack = pickle_unpacker
self.packer = new
else:
self.unpack = import_item(str(new))
def start_kernel(self, **kw):
process_proxy = get_process_proxy_config(self.kernel_spec)
process_proxy_class_name = process_proxy.get('class_name')
self.log.debug(
"Instantiating kernel '{}' with process proxy: {}".format(
self.kernel_spec.display_name, process_proxy_class_name))
process_proxy_class = import_item(process_proxy_class_name)
self.process_proxy = process_proxy_class(
kernel_manager=self, proxy_config=process_proxy.get('config'))
self._capture_user_overrides(**kw)
return super(RemoteKernelManager, self).start_kernel(**kw)
def _get_process_proxy(self):
"""Reads the associated kernelspec and to see if has a process proxy stanza.
If one exists, it instantiates an instance. If a process proxy is not
specified in the kernelspec, a LocalProcessProxy stanza is fabricated and
instantiated.
"""
process_proxy_cfg = get_process_proxy_config(self.kernel_spec)
process_proxy_class_name = process_proxy_cfg.get('class_name')
self.log.debug("Instantiating kernel '{}' with process proxy: {}".
format(self.kernel_spec.display_name, process_proxy_class_name))
process_proxy_class = import_item(process_proxy_class_name)
self.process_proxy = process_proxy_class(kernel_manager=self, proxy_config=process_proxy_cfg.get('config'))
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
sys.stdout.flush()
e_stdout = None if self.quiet else sys.__stdout__
e_stderr = None if self.quiet else sys.__stderr__
sys.stdout = outstream_factory(self.session, self.iopub_thread,
u'stdout',
echo=e_stdout)
sys.stderr.flush()
sys.stderr = outstream_factory(self.session, self.iopub_thread,
u'stderr',
echo=e_stderr)
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
self.displayhook = displayhook_factory(self.session, self.iopub_socket)
sys.displayhook = self.displayhook
self.patch_io()
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp, help = self.subcommands.get(subc)
if isinstance(subapp, six.string_types):
subapp = import_item(subapp)
# clear existing instances
self.__class__.clear_instance()
# instantiate
self.subapp = subapp.instance(config=self.config)
# and initialize subapp
self.subapp.initialize(argv)
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp,help = self.subcommands.get(subc)
if isinstance(subapp, string_types):
subapp = import_item(subapp)
# clear existing instances
self.__class__.clear_instance()
# instantiate
self.subapp = subapp.instance(config=self.config)
# and initialize subapp
self.subapp.initialize(argv)
def _create_kernel_manager_factory(self):
kernel_manager_ctor = import_item(self.kernel_manager_class)
def create_kernel_manager(*args, **kwargs):
if self.shared_context:
if self.context.closed:
# recreate context if closed
self.context = self._context_default()
kwargs.setdefault("context", self.context)
km = kernel_manager_ctor(*args, **kwargs)
return km
return create_kernel_manager
def start_kernel_from_session(self, kernel_id, kernel_name,
connection_info, process_info, launch_args):
# Create a KernelManger instance and load connection and process info, then confirm the kernel is still
# alive.
constructor_kwargs = {}
if self.kernel_spec_manager:
constructor_kwargs[
'kernel_spec_manager'] = self.kernel_spec_manager
# Construct a kernel manager...
km = self.kernel_manager_factory(connection_file=os.path.join(
self.connection_dir, "kernel-%s.json" % kernel_id),
parent=self,
log=self.log,
kernel_name=kernel_name,
**constructor_kwargs)
# Load connection info into member vars - no need to write out connection file
km.load_connection_info(connection_info)
km._launch_args = launch_args
# Construct a process-proxy
if km.kernel_spec.process_proxy_class:
process_proxy_class = import_item(
km.kernel_spec.process_proxy_class)
km.process_proxy = process_proxy_class(
km, proxy_config=km.kernel_spec.process_proxy_config)
km.process_proxy.load_process_info(process_info)
# Confirm we can even poll the process. If not, remove the persisted session.
if km.process_proxy.poll() is False:
return False
km.kernel = km.process_proxy
km.start_restarter()
km._connect_control_socket()
self._kernels[kernel_id] = km
self._kernel_connections[kernel_id] = 0
self.start_watching_activity(kernel_id)
self.add_restart_callback(
kernel_id,
lambda: self._handle_kernel_died(kernel_id),
'dead',
)
# Only initialize culling if available. Warning message will be issued in gatewayapp at startup.
func = getattr(self, 'initialize_culler', None)
if func:
func()
return True
def register_target(self, target_name, f):
"""Register a callable f for a given target name
f will be called with two arguments when a comm_open message is received with `target`:
- the Comm instance
- the `comm_open` message itself.
f can be a Python callable or an import string for one.
"""
if isinstance(f, string_types):
f = import_item(f)
self.targets[target_name] = f
def register_target(self, target_name, f):
"""Register a callable f for a given target name
f will be called with two arguments when a comm_open message is received with `target`:
- the Comm instance
- the `comm_open` message itself.
f can be a Python callable or an import string for one.
"""
if isinstance(f, string_types):
f = import_item(f)
self.targets[target_name] = f
def register_preprocessor(self, preprocessor, enabled=False):
"""
Register a preprocessor.
Preprocessors are classes that act upon the notebook before it is
passed into the Jinja templating engine. preprocessors are also
capable of passing additional information to the Jinja
templating engine.
Parameters
----------
preprocessor : :class:`~nbconvert.preprocessors.Preprocessor`
A dotted module name, a type, or an instance
enabled : bool
Mark the preprocessor as enabled
"""
if preprocessor is None:
raise TypeError("preprocessor")
isclass = isinstance(preprocessor, type)
constructed = not isclass
# Handle preprocessor's registration based on it's type
if constructed and isinstance(preprocessor, py3compat.string_types):
# Preprocessor is a string, import the namespace and recursively call
# this register_preprocessor method
preprocessor_cls = import_item(preprocessor)
return self.register_preprocessor(preprocessor_cls, enabled)
if constructed and hasattr(preprocessor, "__call__"):
# Preprocessor is a function, no need to construct it.
# Register and return the preprocessor.
if enabled:
preprocessor.enabled = True
self._preprocessors.append(preprocessor)
return preprocessor
elif isclass and isinstance(preprocessor, MetaHasTraits):
# Preprocessor is configurable. Make sure to pass in new default for
# the enabled flag if one was specified.
self.register_preprocessor(preprocessor(parent=self), enabled)
elif isclass:
# Preprocessor is not configurable, construct it
self.register_preprocessor(preprocessor(), enabled)
else:
# Preprocessor is an instance of something without a __call__
# attribute.
raise TypeError("preprocessor")
def register_preprocessor(self, preprocessor, enabled=False):
"""
Register a preprocessor.
Preprocessors are classes that act upon the notebook before it is
passed into the Jinja templating engine. preprocessors are also
capable of passing additional information to the Jinja
templating engine.
Parameters
----------
preprocessor : :class:`~nbconvert.preprocessors.Preprocessor`
A dotted module name, a type, or an instance
enabled : bool
Mark the preprocessor as enabled
"""
if preprocessor is None:
raise TypeError('preprocessor')
isclass = isinstance(preprocessor, type)
constructed = not isclass
# Handle preprocessor's registration based on it's type
if constructed and isinstance(preprocessor, py3compat.string_types):
# Preprocessor is a string, import the namespace and recursively call
# this register_preprocessor method
preprocessor_cls = import_item(preprocessor)
return self.register_preprocessor(preprocessor_cls, enabled)
if constructed and hasattr(preprocessor, '__call__'):
# Preprocessor is a function, no need to construct it.
# Register and return the preprocessor.
if enabled:
preprocessor.enabled = True
self._preprocessors.append(preprocessor)
return preprocessor
elif isclass and issubclass(preprocessor, HasTraits):
# Preprocessor is configurable. Make sure to pass in new default for
# the enabled flag if one was specified.
self.register_preprocessor(preprocessor(parent=self), enabled)
elif isclass:
# Preprocessor is not configurable, construct it
self.register_preprocessor(preprocessor(), enabled)
else:
# Preprocessor is an instance of something without a __call__
# attribute.
raise TypeError('preprocessor')
def _import_mapping(mapping, original=None):
"""import any string-keys in a type mapping"""
log = get_logger()
log.debug("Importing canning map")
for key, value in list(mapping.items()):
if isinstance(key, string_types):
try:
cls = import_item(key)
except Exception:
if original and key not in original:
# only message on user-added classes
log.error("canning class not importable: %r", key, exc_info=True)
mapping.pop(key)
else:
mapping[cls] = mapping.pop(key)
def client(self, **kwargs):
"""Create a client configured to connect to our kernel"""
if self.client_factory is None:
self.client_factory = import_item(self.client_class)
kw = {}
kw.update(self.get_connection_info())
kw.update(dict(
connection_file=self.connection_file,
session=self.session,
parent=self,
))
# add kwargs last, for manual overrides
kw.update(kwargs)
return self.client_factory(**kw)
def _create_kernel_manager_factory(self):
kernel_manager_ctor = import_item(self.kernel_manager_class)
def create_kernel_manager(*args, **kwargs):
km = kernel_manager_ctor(*args, **kwargs)
if km.cache_ports:
km.shell_port = self._find_available_port(km.ip)
km.iopub_port = self._find_available_port(km.ip)
km.stdin_port = self._find_available_port(km.ip)
km.hb_port = self._find_available_port(km.ip)
km.control_port = self._find_available_port(km.ip)
return km
return create_kernel_manager
def _import_mapping(mapping, original=None):
"""import any string-keys in a type mapping
"""
log = get_logger()
log.debug("Importing canning map")
for key,value in list(mapping.items()):
if isinstance(key, string_types):
try:
cls = import_item(key)
except Exception:
if original and key not in original:
# only message on user-added classes
log.error("canning class not importable: %r", key, exc_info=True)
mapping.pop(key)
else:
mapping[cls] = mapping.pop(key)
def get_exporter(name):
""" given an exporter name, return a class ready to be instantiate
Raises ValueError if exporter is not found
"""
if name.lower() in exporter_map:
return exporter_map[name.lower()]
if '.' in name:
try:
return import_item(name)
except ImportError:
log = logging.getLogger()
log.error("Error importing %s" % name, exc_info=True)
pass
raise ValueError('Unknown exporter "%s", did you mean one of: %s?'
% (name, ', '.join(sorted(get_export_names()))))
def _register_filter(self, environ, name, jinja_filter):
"""
Register a filter.
A filter is a function that accepts and acts on one string.
The filters are accessible within the Jinja templating engine.
Parameters
----------
name : str
name to give the filter in the Jinja engine
filter : filter
"""
if jinja_filter is None:
raise TypeError('filter')
isclass = isinstance(jinja_filter, type)
constructed = not isclass
#Handle filter's registration based on it's type
if constructed and isinstance(jinja_filter, py3compat.string_types):
#filter is a string, import the namespace and recursively call
#this register_filter method
filter_cls = import_item(jinja_filter)
return self._register_filter(environ, name, filter_cls)
if constructed and hasattr(jinja_filter, '__call__'):
#filter is a function, no need to construct it.
environ.filters[name] = jinja_filter
return jinja_filter
elif isclass and issubclass(jinja_filter, HasTraits):
#filter is configurable. Make sure to pass in new default for
#the enabled flag if one was specified.
filter_instance = jinja_filter(parent=self)
self._register_filter(environ, name, filter_instance)
elif isclass:
#filter is not configurable, construct it
filter_instance = jinja_filter()
self._register_filter(environ, name, filter_instance)
else:
#filter is an instance of something without a __call__
#attribute.
raise TypeError('filter')
def _register_filter(self, environ, name, jinja_filter):
"""
Register a filter.
A filter is a function that accepts and acts on one string.
The filters are accessible within the Jinja templating engine.
Parameters
----------
name : str
name to give the filter in the Jinja engine
filter : filter
"""
if jinja_filter is None:
raise TypeError('filter')
isclass = isinstance(jinja_filter, type)
constructed = not isclass
#Handle filter's registration based on it's type
if constructed and isinstance(jinja_filter, py3compat.string_types):
#filter is a string, import the namespace and recursively call
#this register_filter method
filter_cls = import_item(jinja_filter)
return self._register_filter(environ, name, filter_cls)
if constructed and hasattr(jinja_filter, '__call__'):
#filter is a function, no need to construct it.
environ.filters[name] = jinja_filter
return jinja_filter
elif isclass and issubclass(jinja_filter, HasTraits):
#filter is configurable. Make sure to pass in new default for
#the enabled flag if one was specified.
filter_instance = jinja_filter(parent=self)
self._register_filter(environ, name, filter_instance)
elif isclass:
#filter is not configurable, construct it
filter_instance = jinja_filter()
self._register_filter(environ, name, filter_instance)
else:
#filter is an instance of something without a __call__
#attribute.
raise TypeError('filter')
def get_validator(version=None, version_minor=None, relax_add_props=False):
"""Load the JSON schema into a Validator"""
if version is None:
from . import current_nbformat
version = current_nbformat
v = import_item("nbformat.v%s" % version)
current_minor = getattr(v, 'nbformat_minor', 0)
if version_minor is None:
version_minor = current_minor
current_validator = get_current_validator()
version_tuple = (current_validator.name, version, version_minor)
if version_tuple not in validators:
try:
schema_json = _get_schema_json(v,
version=version,
version_minor=version_minor)
except AttributeError:
return None
if current_minor < version_minor:
# notebook from the future, relax all `additionalProperties: False` requirements
schema_json = _relax_additional_properties(schema_json)
# and allow undefined cell types and outputs
schema_json = _allow_undefined(schema_json)
validators[version_tuple] = current_validator(schema_json)
if relax_add_props:
try:
schema_json = _get_schema_json(v,
version=version,
version_minor=version_minor)
except AttributeError:
return None
# this allows properties to be added for intermediate
# representations while validating for all other kinds of errors
schema_json = _relax_additional_properties(schema_json)
validators[version_tuple] = current_validator(schema_json)
return validators[version_tuple]
def start_kernel_from_session(self, kernel_id, kernel_name, connection_info, process_info, launch_args):
# Create a KernelManger instance and load connection and process info, then confirm the kernel is still
# alive.
constructor_kwargs = {}
if self.kernel_spec_manager:
constructor_kwargs['kernel_spec_manager'] = self.kernel_spec_manager
# Construct a kernel manager...
km = self.kernel_manager_factory(connection_file=os.path.join(
self.connection_dir, "kernel-%s.json" % kernel_id),
parent=self, log=self.log, kernel_name=kernel_name,
**constructor_kwargs)
# Load connection info into member vars - no need to write out connection file
km.load_connection_info(connection_info)
km._launch_args = launch_args
# Construct a process-proxy
process_proxy = get_process_proxy_config(km.kernel_spec)
process_proxy_class = import_item(process_proxy.get('class_name'))
km.process_proxy = process_proxy_class(km, proxy_config=process_proxy.get('config'))
km.process_proxy.load_process_info(process_info)
# Confirm we can even poll the process. If not, remove the persisted session.
if km.process_proxy.poll() is False:
return False
km.kernel = km.process_proxy
km.start_restarter()
km._connect_control_socket()
self._kernels[kernel_id] = km
self._kernel_connections[kernel_id] = 0
self.start_watching_activity(kernel_id)
self.add_restart_callback(kernel_id,
lambda: self._handle_kernel_died(kernel_id),
'dead',
)
# Only initialize culling if available. Warning message will be issued in gatewayapp at startup.
func = getattr(self, 'initialize_culler', None)
if func:
func()
return True
def find_launcher_class(clsname, kind):
"""Return a launcher for a given clsname and kind.
Parameters
==========
clsname : str
The full name of the launcher class, either with or without the
module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor
WindowsHPC).
kind : str
Either 'EngineSet' or 'Controller'.
"""
if '.' not in clsname:
# not a module, presume it's the raw name in apps.launcher
if kind and kind not in clsname:
# doesn't match necessary full class name, assume it's
# just 'PBS' or 'MPI' etc prefix:
clsname = clsname + kind + 'Launcher'
clsname = 'ipyparallel.apps.launcher.'+clsname
klass = import_item(clsname)
return klass
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp, _ = self.subcommands.get(subc)
if isinstance(subapp, str):
subapp = import_item(subapp)
## Cannot issubclass() on a non-type (SOhttp://stackoverflow.com/questions/8692430)
if isinstance(subapp, type) and issubclass(subapp, Application):
# Clear existing instances before...
self.__class__.clear_instance()
# instantiating subapp...
self.subapp = subapp.instance(parent=self)
elif callable(subapp):
# or ask factory to create it...
self.subapp = subapp(self)
else:
raise AssertionError("Invalid mappings for subcommand '%s'!" % subc)
# ... and finally initialize subapp.
self.subapp.initialize(argv)
def find_launcher_class(clsname, kind):
"""Return a launcher for a given clsname and kind.
Parameters
==========
clsname : str
The full name of the launcher class, either with or without the
module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor
Slurm, WindowsHPC).
kind : str
Either 'EngineSet' or 'Controller'.
"""
if '.' not in clsname:
# not a module, presume it's the raw name in apps.launcher
if kind and kind not in clsname:
# doesn't match necessary full class name, assume it's
# just 'PBS' or 'MPI' etc prefix:
clsname = clsname + kind + 'Launcher'
clsname = 'ipyparallel.apps.launcher.' + clsname
klass = import_item(clsname)
return klass
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp, _ = self.subcommands.get(subc)
if isinstance(subapp, six.string_types):
subapp = import_item(subapp)
## Cannot issubclass() on a non-type (SOhttp://stackoverflow.com/questions/8692430)
if isinstance(subapp, type) and issubclass(subapp, Application):
# Clear existing instances before...
self.__class__.clear_instance()
# instantiating subapp...
self.subapp = subapp.instance(parent=self)
elif callable(subapp):
# or ask factory to create it...
self.subapp = subapp(self)
else:
raise AssertionError("Invalid mappings for subcommand '%s'!" % subc)
# ... and finally initialize subapp.
self.subapp.initialize(argv)
def _create_kernel_manager_factory(self):
kernel_manager_ctor = import_item(self.kernel_manager_class)
def create_kernel_manager(*args, **kwargs):
if self.shared_context:
if self.context.closed:
# recreate context if closed
self.context = self._context_default()
kwargs.setdefault("context", self.context)
km = kernel_manager_ctor(*args, **kwargs)
if km.cache_ports:
km.shell_port = self._find_available_port(km.ip)
km.iopub_port = self._find_available_port(km.ip)
km.stdin_port = self._find_available_port(km.ip)
km.hb_port = self._find_available_port(km.ip)
km.control_port = self._find_available_port(km.ip)
return km
return create_kernel_manager
def get(self, bundler_id):
'''
Executes the requested bundler on the given notebook.
:param bundler_id: Unique ID of an installed bundler
:arg notebook: Path to the notebook relative to the notebook directory
root
'''
notebook = self.get_query_argument('notebook')
abs_nb_path = os.path.join(self.notebook_dir, url2path(notebook))
try:
bundler = self.get_bundler(bundler_id)
except KeyError:
raise web.HTTPError(404, 'Bundler %s not found' % bundler_id)
try:
bundler_mod = import_item(bundler['module_name'])
except ImportError:
raise web.HTTPError(500, 'Could not import bundler %s ' % bundler_id)
# Let the bundler respond in any way it sees fit and assume it will
# finish the request
yield gen.maybe_future(bundler_mod.bundle(self, abs_nb_path))
def start_kernel(self, **kwargs):
"""Starts a kernel in a separate process.
Where the started kernel resides depends on the configured process proxy.
Parameters
----------
`**kwargs` : optional
keyword arguments that are passed down to build the kernel_cmd
and launching the kernel (e.g. Popen kwargs).
"""
process_proxy = get_process_proxy_config(self.kernel_spec)
process_proxy_class_name = process_proxy.get('class_name')
self.log.debug(
"Instantiating kernel '{}' with process proxy: {}".format(
self.kernel_spec.display_name, process_proxy_class_name))
process_proxy_class = import_item(process_proxy_class_name)
self.process_proxy = process_proxy_class(
kernel_manager=self, proxy_config=process_proxy.get('config'))
self._capture_user_overrides(**kwargs)
super(RemoteKernelManager, self).start_kernel(**kwargs)
def get_validator(version=None, version_minor=None, relax_add_props=False):
"""Load the JSON schema into a Validator"""
if version is None:
from .. import current_nbformat
version = current_nbformat
v = import_item("nbformat.v%s" % version)
current_minor = getattr(v, 'nbformat_minor', 0)
if version_minor is None:
version_minor = current_minor
version_tuple = (version, version_minor)
if version_tuple not in validators:
try:
schema_json = _get_schema_json(v)
except AttributeError:
return None
if current_minor < version_minor:
# notebook from the future, relax all `additionalProperties: False` requirements
schema_json = _relax_additional_properties(schema_json)
# and allow undefined cell types and outputs
schema_json = _allow_undefined(schema_json)
validators[version_tuple] = Validator(schema_json)
if relax_add_props:
try:
schema_json = _get_schema_json(v)
except AttributeError:
return None
# this allows properties to be added for intermediate
# representations while validating for all other kinds of errors
schema_json = _relax_additional_properties(schema_json)
validators[version_tuple] = Validator(schema_json)
return validators[version_tuple]
def _postprocessor_class_changed(self, name, old, new):
if new.lower() in self.postprocessor_aliases:
new = self.postprocessor_aliases[new.lower()]
if new:
self.postprocessor_factory = import_item(new)
def _writer_class_changed(self, name, old, new):
if new.lower() in self.writer_aliases:
new = self.writer_aliases[new.lower()]
self.writer_factory = import_item(new)
def _post_save_hook_changed(self, name, old, new):
if new and isinstance(new, string_types):
self.post_save_hook = import_item(self.post_save_hook)
elif new:
if not callable(new):
raise TraitError("post_save_hook must be callable")
def _lexer_default(self):
lexer_class = import_item(self.lexer_class)
return lexer_class()
def _lexer_class_changed(self, name, old, new):
lexer_class = import_item(new)
self.lexer = lexer_class()
def init_schedulers(self):
children = self.children
mq = import_item(str(self.mq_class))
f = self.factory
ident = f.session.bsession
# disambiguate url, in case of *
monitor_url = disambiguate_url(f.monitor_url)
# maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
# IOPub relay (in a Process)
q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
q.bind_in(f.client_url('iopub'))
q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub")
q.bind_out(f.engine_url('iopub'))
q.setsockopt_out(zmq.SUBSCRIBE, b'')
q.connect_mon(monitor_url)
q.daemon=True
children.append(q)
# Multiplexer Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
q.bind_in(f.client_url('mux'))
q.setsockopt_in(zmq.IDENTITY, b'mux_in')
q.bind_out(f.engine_url('mux'))
q.setsockopt_out(zmq.IDENTITY, b'mux_out')
q.connect_mon(monitor_url)
q.daemon=True
children.append(q)
# Control Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
q.bind_in(f.client_url('control'))
q.setsockopt_in(zmq.IDENTITY, b'control_in')
q.bind_out(f.engine_url('control'))
q.setsockopt_out(zmq.IDENTITY, b'control_out')
q.connect_mon(monitor_url)
q.daemon=True
children.append(q)
if 'TaskScheduler.scheme_name' in self.config:
scheme = self.config.TaskScheduler.scheme_name
else:
scheme = TaskScheduler.scheme_name.get_default_value()
# Task Queue (in a Process)
if scheme == 'pure':
self.log.warn("task::using pure DEALER Task scheduler")
q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
# q.setsockopt_out(zmq.HWM, hub.hwm)
q.bind_in(f.client_url('task'))
q.setsockopt_in(zmq.IDENTITY, b'task_in')
q.bind_out(f.engine_url('task'))
q.setsockopt_out(zmq.IDENTITY, b'task_out')
q.connect_mon(monitor_url)
q.daemon=True
children.append(q)
elif scheme == 'none':
self.log.warn("task::using no Task scheduler")
else:
self.log.info("task::using Python %s Task scheduler"%scheme)
sargs = (f.client_url('task'), f.engine_url('task'),
monitor_url, disambiguate_url(f.client_url('notification')),
disambiguate_url(f.client_url('registration')),
)
kwargs = dict(logname='scheduler', loglevel=self.log_level,
log_url = self.log_url, config=dict(self.config))
if 'Process' in self.mq_class:
# run the Python scheduler in a Process
q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
q.daemon=True
children.append(q)
else:
# single-threaded Controller
kwargs['in_thread'] = True
launch_scheduler(*sargs, **kwargs)
# set unlimited HWM for all relay devices
if hasattr(zmq, 'SNDHWM'):
q = children[0]
q.setsockopt_in(zmq.RCVHWM, 0)
q.setsockopt_out(zmq.SNDHWM, 0)
for q in children[1:]:
if not hasattr(q, 'setsockopt_in'):
continue
q.setsockopt_in(zmq.SNDHWM, 0)
q.setsockopt_in(zmq.RCVHWM, 0)
q.setsockopt_out(zmq.SNDHWM, 0)
q.setsockopt_out(zmq.RCVHWM, 0)
q.setsockopt_mon(zmq.SNDHWM, 0)
def handle_comm_opened(comm, msg):
"""Static method, called when a widget is constructed."""
widget_class = import_item(str(msg['content']['data']['widget_class']))
widget = widget_class(comm=comm)
def _client_factory_default(self):
return import_item(self.client_class)
def _client_class_changed(self, name, old, new):
self.client_factory = import_item(str(new))
