Ejemplo n.º 1
0
class TransformerWidget(Widget):
    d = List(Bool()).tag(sync=True, from_json=transform_fromjson)
Ejemplo n.º 2
0
class LayerGroup(Layer):
    _view_name = Unicode('LeafletLayerGroupView').tag(sync=True)
    _view_name = Unicode('LeafletLayerGroupModel').tag(sync=True)

    layers = List(Instance(Layer)).tag(sync=True, **widget_serialization)
Ejemplo n.º 3
0
class Map(DOMWidget, InteractMixin):
    @default('layout')
    def _default_layout(self):
        return Layout(height='400px', align_self='stretch')

    _view_name = Unicode('LeafletMapView').tag(sync=True)
    _model_name = Unicode('LeafletMapModel').tag(sync=True)
    _view_module = Unicode('jupyter-leaflet').tag(sync=True)
    _model_module = Unicode('jupyter-leaflet').tag(sync=True)

    # Map options
    center = List(def_loc).tag(sync=True, o=True)
    zoom_start = Int(12).tag(sync=True, o=True)
    zoom = Int(12).tag(sync=True, o=True)
    max_zoom = Int(18).tag(sync=True, o=True)
    min_zoom = Int(1).tag(sync=True, o=True)
    # Interaction options
    dragging = Bool(True).tag(sync=True, o=True)
    touch_zoom = Bool(True).tag(sync=True, o=True)
    scroll_wheel_zoom = Bool(False).tag(sync=True, o=True)
    double_click_zoom = Bool(True).tag(sync=True, o=True)
    box_zoom = Bool(True).tag(sync=True, o=True)
    tap = Bool(True).tag(sync=True, o=True)
    tap_tolerance = Int(15).tag(sync=True, o=True)
    world_copy_jump = Bool(False).tag(sync=True, o=True)
    close_popup_on_click = Bool(True).tag(sync=True, o=True)
    bounce_at_zoom_limits = Bool(True).tag(sync=True, o=True)
    keyboard = Bool(True).tag(sync=True, o=True)
    keyboard_pan_offset = Int(80).tag(sync=True, o=True)
    keyboard_zoom_offset = Int(1).tag(sync=True, o=True)
    inertia = Bool(True).tag(sync=True, o=True)
    inertia_deceleration = Int(3000).tag(sync=True, o=True)
    inertia_max_speed = Int(1500).tag(sync=True, o=True)
    # inertia_threshold = Int(?, o=True).tag(sync=True)
    zoom_control = Bool(True).tag(sync=True, o=True)
    attribution_control = Bool(True).tag(sync=True, o=True)
    # fade_animation = Bool(?).tag(sync=True, o=True)
    # zoom_animation = Bool(?).tag(sync=True, o=True)
    zoom_animation_threshold = Int(4).tag(sync=True, o=True)
    # marker_zoom_animation = Bool(?).tag(sync=True, o=True)

    options = List(trait=Unicode).tag(sync=True)

    @default('options')
    def _default_options(self):
        return [name for name in self.traits(o=True)]

    _south = Float(def_loc[0]).tag(sync=True)
    _north = Float(def_loc[0]).tag(sync=True)
    _east = Float(def_loc[1]).tag(sync=True)
    _west = Float(def_loc[1]).tag(sync=True)

    default_tiles = Instance(TileLayer, allow_none=True)

    @default('default_tiles')
    def _default_tiles(self):
        return TileLayer()

    @property
    def north(self):
        return self._north

    @property
    def south(self):
        return self._south

    @property
    def east(self):
        return self._east

    @property
    def west(self):
        return self._west

    @property
    def bounds_polygon(self):
        return [(self.north, self.west), (self.north, self.east),
                (self.south, self.east), (self.south, self.west)]

    @property
    def bounds(self):
        return [(self.south, self.west), (self.north, self.east)]

    def __init__(self, **kwargs):
        super(Map, self).__init__(**kwargs)
        self.on_displayed(self._fire_children_displayed)
        if self.default_tiles is not None:
            self.layers = (self.default_tiles, )
        self.on_msg(self._handle_leaflet_event)

    def _fire_children_displayed(self, widget, **kwargs):
        for layer in self.layers:
            layer._handle_displayed(**kwargs)
        for control in self.controls:
            control._handle_displayed(**kwargs)

    layers = Tuple(trait=Instance(Layer)).tag(sync=True,
                                              **widget_serialization)
    layer_ids = List()

    @validate('layers')
    def _validate_layers(self, proposal):
        """Validate layers list.

        Makes sure only one instance of any given layer can exist in the
        layers list.
        """
        self.layer_ids = [l.model_id for l in proposal['value']]
        if len(set(self.layer_ids)) != len(self.layer_ids):
            raise LayerException(
                'duplicate layer detected, only use each layer once')
        return proposal['value']

    def add_layer(self, layer):
        if layer.model_id in self.layer_ids:
            raise LayerException('layer already on map: %r' % layer)
        layer._map = self
        self.layers = tuple([l for l in self.layers] + [layer])
        layer.visible = True

    def remove_layer(self, layer):
        if layer.model_id not in self.layer_ids:
            raise LayerException('layer not on map: %r' % layer)
        self.layers = tuple(
            [l for l in self.layers if l.model_id != layer.model_id])
        layer.visible = False

    def clear_layers(self):
        self.layers = ()

    controls = Tuple(trait=Instance(Control)).tag(sync=True,
                                                  **widget_serialization)
    control_ids = List()

    @validate('controls')
    def _validate_controls(self, proposal):
        """Validate controls list.

        Makes sure only one instance of any given layer can exist in the
        controls list.
        """
        self.control_ids = [c.model_id for c in proposal['value']]
        if len(set(self.control_ids)) != len(self.control_ids):
            raise ControlException(
                'duplicate control detected, only use each control once')
        return proposal['value']

    def add_control(self, control):
        if control.model_id in self.control_ids:
            raise ControlException('control already on map: %r' % control)
        control._map = self
        self.controls = tuple([c for c in self.controls] + [control])
        control.visible = True

    def remove_control(self, control):
        if control.model_id not in self.control_ids:
            raise ControlException('control not on map: %r' % control)
        self.controls = tuple(
            [c for c in self.controls if c.model_id != control.model_id])
        control.visible = False

    def clear_controls(self):
        self.controls = ()

    def __iadd__(self, item):
        if isinstance(item, Layer):
            self.add_layer(item)
        elif isinstance(item, Control):
            self.add_control(item)
        return self

    def __isub__(self, item):
        if isinstance(item, Layer):
            self.remove_layer(item)
        elif isinstance(item, Control):
            self.remove_control(item)
        return self

    def __add__(self, item):
        if isinstance(item, Layer):
            self.add_layer(item)
        elif isinstance(item, Control):
            self.add_control(item)
        return self

    def _handle_leaflet_event(self, _, content):
        pass
Ejemplo n.º 4
0
class Drawing(GMapsWidgetMixin, widgets.Widget):
    """
    Widget for a drawing layer

    Add this to a :class:`gmaps.Map` or :class:`gmaps.Figure` instance to let
    you draw on the map.

    You should not need to instantiate this directly. Instead, use the
    :func:`gmaps.drawing_layer` factory function.

    :Examples:

    {examples}

    {params}

    :param mode:
        Initial drawing mode. One of ``DISABLED``, ``MARKER``, ``LINE``,
        ``POLYGON`` or ``DELETE``. Defaults to ``MARKER`` if
        ``toolbar_controls.show_controls`` is True, otherwise defaults to
        ``DISABLED``.
    :type mode: str, optional

    :param toolbar_controls:
        Widget representing the drawing toolbar.
    :type toolbar_controls: :class:`gmaps.DrawingControls`, optional
    """
    has_bounds = False
    _view_name = Unicode('DrawingLayerView').tag(sync=True)
    _model_name = Unicode('DrawingLayerModel').tag(sync=True)
    features = List().tag(sync=True, **widgets.widget_serialization)
    mode = Enum(ALLOWED_DRAWING_MODES).tag(sync=True)
    marker_options = widgets.trait_types.InstanceDict(MarkerOptions,
                                                      allow_none=False)
    line_options = widgets.trait_types.InstanceDict(LineOptions,
                                                    allow_none=False)
    toolbar_controls = Instance(DrawingControls, allow_none=False).tag(
        sync=True, **widgets.widget_serialization)

    def __init__(self, **kwargs):
        kwargs['mode'] = self._get_initial_mode(kwargs)
        if kwargs.get('features') is None:
            kwargs['features'] = []
        if kwargs.get('marker_options') is None:
            kwargs['marker_options'] = self._default_marker_options()
        if kwargs.get('line_options') is None:
            kwargs['line_options'] = self._default_line_options()
        self._new_feature_callbacks = []

        super(Drawing, self).__init__(**kwargs)
        self.on_msg(self._handle_message)

    def on_new_feature(self, callback):
        """
        Register a callback called when new features are added

        :param callback:
            Callable to be called when a new feature is added.
            The callback should take a single argument, the
            feature that has been added. This can be an instance
            of :class:`gmaps.Line`, :class:`gmaps.Marker` or
            :class:`gmaps.Polygon`.
        :type callback: callable
        """
        self._new_feature_callbacks.append(callback)

    def _get_initial_mode(self, constructor_kwargs):
        try:
            mode = constructor_kwargs['mode']
        except KeyError:
            # mode not explicitly specified
            controls_hidden = (
                'toolbar_controls' in constructor_kwargs
                and not constructor_kwargs['toolbar_controls'].show_controls)
            if controls_hidden:
                mode = 'DISABLED'
            else:
                mode = DEFAULT_DRAWING_MODE
        return mode

    @default('marker_options')
    def _default_marker_options(self):
        return MarkerOptions()

    @default('line_options')
    def _default_line_options(self):
        return LineOptions()

    @default('toolbar_controls')
    def _default_toolbar_controls(self):
        return DrawingControls()

    @observe('features')
    def _on_new_feature(self, change):
        if self._new_feature_callbacks:
            old_features = set(change['old'])
            new_features = [
                feature for feature in change['new']
                if feature not in old_features
            ]
            for feature in new_features:
                for callback in self._new_feature_callbacks:
                    callback(feature)

    def _delete_feature(self, model_id):
        updated_features = [
            feature for feature in self.features
            if feature.model_id != model_id
        ]
        self.features = updated_features

    def _handle_message(self, _, content, buffers):
        if content.get('event') == 'FEATURE_ADDED':
            payload = content['payload']
            if payload['featureType'] == 'MARKER':
                latitude = payload['latitude']
                longitude = payload['longitude']
                feature = self.marker_options.to_marker(latitude, longitude)
            elif payload['featureType'] == 'LINE':
                start = payload['start']
                end = payload['end']
                feature = self.line_options.to_line(start, end)
            elif payload['featureType'] == 'POLYGON':
                path = payload['path']
                feature = Polygon(path)
            self.features = self.features + [feature]
        elif content.get('event') == 'MODE_CHANGED':
            payload = content['payload']
            mode = payload['mode']
            self.mode = mode
        elif content.get('event') == 'FEATURE_DELETED':
            payload = content['payload']
            model_id = payload['modelId']
            self._delete_feature(model_id)
Ejemplo n.º 5
0
class Rectangle(Polygon):
    _view_name = Unicode('LeafletRectangleView').tag(sync=True)
    _model_name = Unicode('LeafletRectangleModel').tag(sync=True)

    bounds = List(help="list of SW and NE location tuples").tag(sync=True)
Ejemplo n.º 6
0
class Layer(Widget, InteractMixin):
    _view_name = Unicode('LeafletLayerView').tag(sync=True)
    _model_name = Unicode('LeafletLayerModel').tag(sync=True)
    _view_module = Unicode('jupyter-leaflet').tag(sync=True)
    _model_module = Unicode('jupyter-leaflet').tag(sync=True)

    _view_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
    _model_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)

    name = Unicode('').tag(sync=True)
    base = Bool(False).tag(sync=True)
    bottom = Bool(False).tag(sync=True)
    popup = Instance(Widget, allow_none=True,
                     default_value=None).tag(sync=True, **widget_serialization)
    popup_min_width = Int(50).tag(sync=True)
    popup_max_width = Int(300).tag(sync=True)
    popup_max_height = Int(default_value=None, allow_none=True).tag(sync=True)

    options = List(trait=Unicode()).tag(sync=True)

    def __init__(self, **kwargs):
        super(Layer, self).__init__(**kwargs)
        self.on_msg(self._handle_mouse_events)

    @default('options')
    def _default_options(self):
        return [name for name in self.traits(o=True)]

    # Event handling
    _click_callbacks = Instance(CallbackDispatcher, ())
    _dblclick_callbacks = Instance(CallbackDispatcher, ())
    _mousedown_callbacks = Instance(CallbackDispatcher, ())
    _mouseup_callbacks = Instance(CallbackDispatcher, ())
    _mouseover_callbacks = Instance(CallbackDispatcher, ())
    _mouseout_callbacks = Instance(CallbackDispatcher, ())

    def _handle_mouse_events(self, _, content, buffers):
        event_type = content.get('type', '')
        if event_type == 'click':
            self._click_callbacks(**content)
        if event_type == 'dblclick':
            self._dblclick_callbacks(**content)
        if event_type == 'mousedown':
            self._mousedown_callbacks(**content)
        if event_type == 'mouseup':
            self._mouseup_callbacks(**content)
        if event_type == 'mouseover':
            self._mouseover_callbacks(**content)
        if event_type == 'mouseout':
            self._mouseout_callbacks(**content)

    def on_click(self, callback, remove=False):
        self._click_callbacks.register_callback(callback, remove=remove)

    def on_dblclick(self, callback, remove=False):
        self._dblclick_callbacks.register_callback(callback, remove=remove)

    def on_mousedown(self, callback, remove=False):
        self._mousedown_callbacks.register_callback(callback, remove=remove)

    def on_mouseup(self, callback, remove=False):
        self._mouseup_callbacks.register_callback(callback, remove=remove)

    def on_mouseover(self, callback, remove=False):
        self._mouseover_callbacks.register_callback(callback, remove=remove)

    def on_mouseout(self, callback, remove=False):
        self._mouseout_callbacks.register_callback(callback, remove=remove)
Ejemplo n.º 7
0
class SwarmSpawner(Spawner):
    """A Spawner for JupyterHub using Docker Engine in Swarm mode
    """

    _executor = None

    @property
    def executor(self, max_workers=1):
        """single global executor"""
        cls = self.__class__
        if cls._executor is None:
            cls._executor = ThreadPoolExecutor(max_workers)
        return cls._executor

    _client = None

    @property
    def client(self):
        """single global client instance"""
        cls = self.__class__

        if cls._client is None:
            kwargs = {}
            if self.tls_config:
                kwargs['tls'] = docker.tls.TLSConfig(**self.tls_config)
            kwargs.update(kwargs_from_env())
            client = docker.APIClient(version='auto', **kwargs)

            cls._client = client
        return cls._client

    service_id = Unicode()
    service_port = Int(8888, min=1, max=65535, config=True)
    service_image = Unicode("jupyterhub/singleuser", config=True)
    service_prefix = Unicode("jupyter",
                             config=True,
                             help=dedent("""
            Prefix for service names. The full service name for a particular
            user will be <prefix>-<hash(username)>-<server_name>.
            """))
    tls_config = Dict(
        config=True,
        help=dedent("""Arguments to pass to docker TLS configuration.
            Check for more info: http://docker-py.readthedocs.io/en/stable/tls.html
            """))

    container_spec = Dict({}, config=True, help="Params to create the service")
    resource_spec = Dict({},
                         config=True,
                         help="Params about cpu and memory limits")

    placement = List(
        [],
        config=True,
        help=dedent("""List of placement constraints into the swarm
                         """))

    networks = List(
        [],
        config=True,
        help=dedent("""Additional args to create_host_config for service create
                        """))
    use_user_options = Bool(
        False,
        config=True,
        help=dedent("""the spawner will use the dict passed through the form
                                or as json body when using the Hub Api
                                """))
    jupyterhub_service_name = Unicode(
        config=True,
        help=dedent("""Name of the service running the JupyterHub
                                          """))

    @property
    def tls_client(self):
        """A tuple consisting of the TLS client certificate and key if they
        have been provided, otherwise None.

        """
        if self.tls_cert and self.tls_key:
            return (self.tls_cert, self.tls_key)
        return None

    _service_owner = None

    @property
    def service_owner(self):
        if self._service_owner is None:
            m = hashlib.md5()
            m.update(self.user.name.encode('utf-8'))
            self._service_owner = m.hexdigest()
        return self._service_owner

    @property
    def service_name(self):
        """
        Service name inside the Docker Swarm

        service_suffix should be a numerical value unique for user
        {service_prefix}-{service_owner}-{service_suffix}
        """
        if hasattr(self, "server_name") and self.server_name:
            server_name = self.server_name
        else:
            server_name = 1

        return "{}-{}-{}".format(self.service_prefix, self.service_owner,
                                 server_name)

    def load_state(self, state):
        super().load_state(state)
        self.service_id = state.get('service_id', '')

    def get_state(self):
        state = super().get_state()
        if self.service_id:
            state['service_id'] = self.service_id
        return state

    def _env_keep_default(self):
        """it's called in traitlets. It's a special method name.
        Don't inherit any env from the parent process"""
        return []

    def _public_hub_api_url(self):
        proto, path = self.hub.api_url.split('://', 1)
        _, rest = path.split(':', 1)
        return '{proto}://{name}:{rest}'.format(
            proto=proto, name=self.jupyterhub_service_name, rest=rest)

    def get_env(self):
        env = super().get_env()
        env.update(
            dict(JPY_USER=self.user.name,
                 JPY_COOKIE_NAME=self.user.server.cookie_name,
                 JPY_BASE_URL=self.user.server.base_url,
                 JPY_HUB_PREFIX=self.hub.server.base_url))

        if self.notebook_dir:
            env['NOTEBOOK_DIR'] = self.notebook_dir

        env['JPY_HUB_API_URL'] = self._public_hub_api_url()

        return env

    def _docker(self, method, *args, **kwargs):
        """wrapper for calling docker methods

        to be passed to ThreadPoolExecutor
        """
        m = getattr(self.client, method)
        return m(*args, **kwargs)

    def docker(self, method, *args, **kwargs):
        """Call a docker method in a background thread

        returns a Future
        """
        return self.executor.submit(self._docker, method, *args, **kwargs)

    @gen.coroutine
    def poll(self):
        """Check for a task state like `docker service ps id`"""
        service = yield self.get_service()
        if not service:
            self.log.warn("Docker service not found")
            return 0

        task_filter = {'service': service['Spec']['Name']}

        tasks = yield self.docker('tasks', task_filter)

        running_task = None
        for task in tasks:
            task_state = task['Status']['State']
            self.log.debug(
                "Task %s of Docker service %s status: %s",
                task['ID'][:7],
                self.service_id[:7],
                pformat(task_state),
            )
            if task_state == 'running':
                # there should be at most one running task
                running_task = task

        if running_task is not None:
            return None
        else:
            return 1

    @gen.coroutine
    def get_service(self):
        self.log.debug("Getting Docker service '%s'", self.service_name)
        try:
            service = yield self.docker('inspect_service', self.service_name)
            self.service_id = service['ID']
        except APIError as err:
            if err.response.status_code == 404:
                self.log.info("Docker service '%s' is gone", self.service_name)
                service = None
                # Docker service is gone, remove service id
                self.service_id = ''
            elif err.response.status_code == 500:
                self.log.info("Docker Swarm Server error")
                service = None
                # Docker service is unhealthy, remove the service_id
                self.service_id = ''
            else:
                raise
        return service

    @gen.coroutine
    def start(self):
        """Start the single-user server in a docker service.
        You can specify the params for the service through jupyterhub_config.py
        or using the user_options
        """

        # https://github.com/jupyterhub/jupyterhub/blob/master/jupyterhub/user.py#L202
        # By default jupyterhub calls the spawner passing user_options
        if self.use_user_options:
            user_options = self.user_options
        else:
            user_options = {}

        self.log.warn("user_options: {}".format(user_options))

        service = yield self.get_service()

        if service is None:

            if 'name' in user_options:
                self.server_name = user_options['name']

            if hasattr(self,
                       'container_spec') and self.container_spec is not None:
                container_spec = dict(**self.container_spec)
            elif user_options == {}:
                raise ("A container_spec is needed in to create a service")

            container_spec.update(user_options.get('container_spec', {}))

            # iterates over mounts to create
            # a new mounts list of docker.types.Mount
            container_spec['mounts'] = []
            for mount in self.container_spec['mounts']:
                m = dict(**mount)

                if 'source' in m:
                    m['source'] = m['source'].format(
                        username=self.service_owner)

                if 'driver_config' in m:
                    device = m['driver_config']['options']['device'].format(
                        username=self.service_owner)
                    m['driver_config']['options']['device'] = device
                    m['driver_config'] = docker.types.DriverConfig(
                        **m['driver_config'])

                container_spec['mounts'].append(docker.types.Mount(**m))

            # some Envs are required by the single-user-image
            container_spec['env'] = self.get_env()

            if hasattr(self, 'resource_spec'):
                resource_spec = dict(**self.resource_spec)
            resource_spec.update(user_options.get('resource_spec', {}))
            # enable to set a human readable memory unit
            if 'mem_limit' in resource_spec:
                resource_spec['mem_limit'] = parse_bytes(
                    resource_spec['mem_limit'])
            if 'mem_reservation' in resource_spec:
                resource_spec['mem_reservation'] = parse_bytes(
                    resource_spec['mem_reservation'])

            if hasattr(self, 'networks'):
                networks = self.networks
            if user_options.get('networks') is not None:
                networks = user_options.get('networks')

            if hasattr(self, 'placement'):
                placement = self.placement
            if user_options.get('placement') is not None:
                placement = user_options.get('placement')

            image = container_spec['Image']
            del container_spec['Image']

            # create the service
            container_spec = docker.types.ContainerSpec(
                image, **container_spec)
            resources = docker.types.Resources(**resource_spec)

            task_spec = {
                'container_spec': container_spec,
                'resources': resources,
                'placement': placement
            }
            task_tmpl = docker.types.TaskTemplate(**task_spec)

            resp = yield self.docker('create_service',
                                     task_tmpl,
                                     name=self.service_name,
                                     networks=networks)

            self.service_id = resp['ID']

            self.log.info("Created Docker service '%s' (id: %s) from image %s",
                          self.service_name, self.service_id[:7], image)

        else:
            self.log.info("Found existing Docker service '%s' (id: %s)",
                          self.service_name, self.service_id[:7])
            # Handle re-using API token.
            # Get the API token from the environment variables
            # of the running service:
            envs = service['Spec']['TaskTemplate']['ContainerSpec']['Env']
            for line in envs:
                if line.startswith('JPY_API_TOKEN='):
                    self.api_token = line.split('=', 1)[1]
                    break

        ip = self.service_name
        port = self.service_port

        # we use service_name instead of ip
        # https://docs.docker.com/engine/swarm/networking/#use-swarm-mode-service-discovery
        # service_port is actually equal to 8888
        return (ip, port)

    @gen.coroutine
    def stop(self, now=False):
        """Stop and remove the service

        Consider using stop/start when Docker adds support
        """
        self.log.info("Stopping and removing Docker service %s (id: %s)",
                      self.service_name, self.service_id[:7])
        yield self.docker('remove_service', self.service_id[:7])
        self.log.info("Docker service %s (id: %s) removed", self.service_name,
                      self.service_id[:7])

        self.clear_state()
Ejemplo n.º 8
0
class DL1Extractor(Tool):
    name = "DL1Extractor"
    description = "Extract the dl1 information and store into a numpy file"

    aliases = Dict(dict(r='EventFileReaderFactory.reader',
                        f='EventFileReaderFactory.input_path',
                        max_events='EventFileReaderFactory.max_events',
                        ped='CameraR1CalibratorFactory.pedestal_path',
                        tf='CameraR1CalibratorFactory.tf_path',
                        pe='CameraR1CalibratorFactory.pe_path',
                        ))
    classes = List([EventFileReaderFactory,
                    CameraR1CalibratorFactory,
                    ])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.reader = None
        self.r1 = None
        self.dl0 = None

        self.output_dir = None

        self.baseline_rms_full = None

    def setup(self):
        self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"
        kwargs = dict(config=self.config, tool=self)

        reader_factory = EventFileReaderFactory(**kwargs)
        reader_class = reader_factory.get_class()
        self.reader = reader_class(**kwargs)

        r1_factory = CameraR1CalibratorFactory(origin=self.reader.origin,
                                               **kwargs)
        r1_class = r1_factory.get_class()
        self.r1 = r1_class(**kwargs)

        self.dl0 = CameraDL0Reducer(**kwargs)

        self.output_dir = join(self.reader.output_directory, "extract_adc2pe")
        if not exists(self.output_dir):
            self.log.info("Creating directory: {}".format(self.output_dir))
            makedirs(self.output_dir)

        n_events = self.reader.num_events
        first_event = self.reader.get_event(0)
        n_pixels = first_event.inst.num_pixels[0]
        n_samples = first_event.r0.tel[0].num_samples

        self.baseline_rms_full = np.zeros((n_events, n_pixels))

    def start(self):
        n_events = self.reader.num_events
        first_event = self.reader.get_event(0)
        telid = list(first_event.r0.tels_with_data)[0]
        n_pixels = first_event.inst.num_pixels[0]
        n_samples = first_event.r0.tel[0].num_samples

        source = self.reader.read()
        desc = "Looping through file"
        with tqdm(total=n_events, desc=desc) as pbar:
            for event in source:
                pbar.update(1)
                ev = event.count

                self.r1.calibrate(event)
                self.dl0.reduce(event)

                dl0 = event.dl0.tel[telid].pe_samples[0]

                baseline_rms_full = np.std(dl0, axis=1)

                self.baseline_rms_full[ev] = baseline_rms_full

    def finish(self):
        output_path = self.reader.input_path.replace("_r0.tio", "_rms.npz")
        output_path = output_path.replace("_r1.tio", "_rms.npy")
        np.save(output_path, self.baseline_rms_full)
        self.log.info("RMS Numpy array saved to: {}".format(output_path))
Ejemplo n.º 9
0
class HistoryManager(HistoryAccessor):
    """A class to organize all history-related functionality in one place.
    """
    # Public interface

    # An instance of the IPython shell we are attached to
    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                     allow_none=True)
    # Lists to hold processed and raw history. These start with a blank entry
    # so that we can index them starting from 1
    input_hist_parsed = List([""])
    input_hist_raw = List([""])
    # A list of directories visited during session
    dir_hist = List()
    @default('dir_hist')
    def _dir_hist_default(self):
        try:
            return [os.getcwd()]
        except OSError:
            return []

    # A dict of output history, keyed with ints from the shell's
    # execution count.
    output_hist = Dict()
    # The text/plain repr of outputs.
    output_hist_reprs = Dict()

    # The number of the current session in the history database
    session_number = Integer()
    
    db_log_output = Bool(False,
        help="Should the history database include output? (default: no)"
    ).tag(config=True)
    db_cache_size = Integer(0,
        help="Write to database every x commands (higher values save disk access & power).\n"
        "Values of 1 or less effectively disable caching."
    ).tag(config=True)
    # The input and output caches
    db_input_cache = List()
    db_output_cache = List()
    
    # History saving in separate thread
    save_thread = Instance('IPython.core.history.HistorySavingThread',
                           allow_none=True)
    save_flag = Instance(threading.Event, allow_none=True)
    
    # Private interface
    # Variables used to store the three last inputs from the user.  On each new
    # history update, we populate the user's namespace with these, shifted as
    # necessary.
    _i00 = Unicode(u'')
    _i = Unicode(u'')
    _ii = Unicode(u'')
    _iii = Unicode(u'')

    # A regex matching all forms of the exit command, so that we don't store
    # them in the history (it's annoying to rewind the first entry and land on
    # an exit call).
    _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")

    def __init__(self, shell=None, config=None, **traits):
        """Create a new history manager associated with a shell instance.
        """
        # We need a pointer back to the shell for various tasks.
        super(HistoryManager, self).__init__(shell=shell, config=config,
            **traits)
        self.save_flag = threading.Event()
        self.db_input_cache_lock = threading.Lock()
        self.db_output_cache_lock = threading.Lock()
        
        try:
            self.new_session()
        except OperationalError:
            self.log.error("Failed to create history session in %s. History will not be saved.",
                self.hist_file, exc_info=True)
            self.hist_file = ':memory:'
        
        if self.enabled and self.hist_file != ':memory:':
            self.save_thread = HistorySavingThread(self)
            self.save_thread.start()

    def _get_hist_file_name(self, profile=None):
        """Get default history file name based on the Shell's profile.
        
        The profile parameter is ignored, but must exist for compatibility with
        the parent class."""
        profile_dir = self.shell.profile_dir.location
        return os.path.join(profile_dir, 'history.sqlite')
    
    @needs_sqlite
    def new_session(self, conn=None):
        """Get a new session number."""
        if conn is None:
            conn = self.db
        
        with conn:
            cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
                            NULL, "") """, (datetime.datetime.now(),))
            self.session_number = cur.lastrowid
            
    def end_session(self):
        """Close the database session, filling in the end time and line count."""
        self.writeout_cache()
        with self.db:
            self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
                            session==?""", (datetime.datetime.now(),
                            len(self.input_hist_parsed)-1, self.session_number))
        self.session_number = 0
                            
    def name_session(self, name):
        """Give the current session a name in the history database."""
        with self.db:
            self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
                            (name, self.session_number))
                            
    def reset(self, new_session=True):
        """Clear the session history, releasing all object references, and
        optionally open a new session."""
        self.output_hist.clear()
        # The directory history can't be completely empty
        self.dir_hist[:] = [os.getcwd()]
        
        if new_session:
            if self.session_number:
                self.end_session()
            self.input_hist_parsed[:] = [""]
            self.input_hist_raw[:] = [""]
            self.new_session()

    # ------------------------------
    # Methods for retrieving history
    # ------------------------------
    def get_session_info(self, session=0):
        """Get info about a session.

        Parameters
        ----------

        session : int
            Session number to retrieve. The current session is 0, and negative
            numbers count back from current session, so -1 is the previous session.

        Returns
        -------
        
        session_id : int
           Session ID number
        start : datetime
           Timestamp for the start of the session.
        end : datetime
           Timestamp for the end of the session, or None if IPython crashed.
        num_cmds : int
           Number of commands run, or None if IPython crashed.
        remark : unicode
           A manually set description.
        """
        if session <= 0:
            session += self.session_number

        return super(HistoryManager, self).get_session_info(session=session)

    def _get_range_session(self, start=1, stop=None, raw=True, output=False):
        """Get input and output history from the current session. Called by
        get_range, and takes similar parameters."""
        input_hist = self.input_hist_raw if raw else self.input_hist_parsed
            
        n = len(input_hist)
        if start < 0:
            start += n
        if not stop or (stop > n):
            stop = n
        elif stop < 0:
            stop += n
        
        for i in range(start, stop):
            if output:
                line = (input_hist[i], self.output_hist_reprs.get(i))
            else:
                line = input_hist[i]
            yield (0, i, line)
    
    def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
        """Retrieve input by session.
        
        Parameters
        ----------
        session : int
            Session number to retrieve. The current session is 0, and negative
            numbers count back from current session, so -1 is previous session.
        start : int
            First line to retrieve.
        stop : int
            End of line range (excluded from output itself). If None, retrieve
            to the end of the session.
        raw : bool
            If True, return untranslated input
        output : bool
            If True, attempt to include output. This will be 'real' Python
            objects for the current session, or text reprs from previous
            sessions if db_log_output was enabled at the time. Where no output
            is found, None is used.
            
        Returns
        -------
        entries
          An iterator over the desired lines. Each line is a 3-tuple, either
          (session, line, input) if output is False, or
          (session, line, (input, output)) if output is True.
        """
        if session <= 0:
            session += self.session_number
        if session==self.session_number:          # Current session
            return self._get_range_session(start, stop, raw, output)
        return super(HistoryManager, self).get_range(session, start, stop, raw,
                                                     output)

    ## ----------------------------
    ## Methods for storing history:
    ## ----------------------------
    def store_inputs(self, line_num, source, source_raw=None):
        """Store source and raw input in history and create input cache
        variables ``_i*``.

        Parameters
        ----------
        line_num : int
          The prompt number of this input.

        source : str
          Python input.

        source_raw : str, optional
          If given, this is the raw input without any IPython transformations
          applied to it.  If not given, ``source`` is used.
        """
        if source_raw is None:
            source_raw = source
        source = source.rstrip('\n')
        source_raw = source_raw.rstrip('\n')

        # do not store exit/quit commands
        if self._exit_re.match(source_raw.strip()):
            return

        self.input_hist_parsed.append(source)
        self.input_hist_raw.append(source_raw)

        with self.db_input_cache_lock:
            self.db_input_cache.append((line_num, source, source_raw))
            # Trigger to flush cache and write to DB.
            if len(self.db_input_cache) >= self.db_cache_size:
                self.save_flag.set()

        # update the auto _i variables
        self._iii = self._ii
        self._ii = self._i
        self._i = self._i00
        self._i00 = source_raw

        # hackish access to user namespace to create _i1,_i2... dynamically
        new_i = '_i%s' % line_num
        to_main = {'_i': self._i,
                   '_ii': self._ii,
                   '_iii': self._iii,
                   new_i : self._i00 }
        
        if self.shell is not None:
            self.shell.push(to_main, interactive=False)

    def store_output(self, line_num):
        """If database output logging is enabled, this saves all the
        outputs from the indicated prompt number to the database. It's
        called by run_cell after code has been executed.

        Parameters
        ----------
        line_num : int
          The line number from which to save outputs
        """
        if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
            return
        output = self.output_hist_reprs[line_num]

        with self.db_output_cache_lock:
            self.db_output_cache.append((line_num, output))
        if self.db_cache_size <= 1:
            self.save_flag.set()

    def _writeout_input_cache(self, conn):
        with conn:
            for line in self.db_input_cache:
                conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
                                (self.session_number,)+line)

    def _writeout_output_cache(self, conn):
        with conn:
            for line in self.db_output_cache:
                conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
                                (self.session_number,)+line)

    @needs_sqlite
    def writeout_cache(self, conn=None):
        """Write any entries in the cache to the database."""
        if conn is None:
            conn = self.db

        with self.db_input_cache_lock:
            try:
                self._writeout_input_cache(conn)
            except sqlite3.IntegrityError:
                self.new_session(conn)
                print("ERROR! Session/line number was not unique in",
                      "database. History logging moved to new session",
                                                self.session_number)
                try:
                    # Try writing to the new session. If this fails, don't
                    # recurse
                    self._writeout_input_cache(conn)
                except sqlite3.IntegrityError:
                    pass
            finally:
                self.db_input_cache = []

        with self.db_output_cache_lock:
            try:
                self._writeout_output_cache(conn)
            except sqlite3.IntegrityError:
                print("!! Session/line number for output was not unique",
                      "in database. Output will not be stored.")
            finally:
                self.db_output_cache = []
Ejemplo n.º 10
0
class IPEngineApp(BaseParallelApplication):

    name = 'ipengine'
    description = _description
    examples = _examples
    classes = List(
        [ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI])

    startup_script = Unicode(u'',
                             config=True,
                             help='specify a script to be run at startup')
    startup_command = Unicode('',
                              config=True,
                              help='specify a command to be run at startup')

    url_file = Unicode(
        u'',
        config=True,
        help=
        """The full location of the file containing the connection information for
        the controller. If this is not given, the file must be in the
        security directory of the cluster directory.  This location is
        resolved using the `profile` or `profile_dir` options.""",
    )
    wait_for_url_file = Float(
        5,
        config=True,
        help="""The maximum number of seconds to wait for url_file to exist.
        This is useful for batch-systems and shared-filesystems where the
        controller and engine are started at the same time and it
        may take a moment for the controller to write the connector files.""")

    url_file_name = Unicode(u'ipcontroller-engine.json', config=True)

    def _cluster_id_changed(self, name, old, new):
        if new:
            base = 'ipcontroller-%s' % new
        else:
            base = 'ipcontroller'
        self.url_file_name = "%s-engine.json" % base

    log_url = Unicode(
        '',
        config=True,
        help="""The URL for the iploggerapp instance, for forwarding
        logging to a central location.""")

    # an IPKernelApp instance, used to setup listening for shell frontends
    kernel_app = Instance(IPKernelApp, allow_none=True)

    aliases = Dict(aliases)
    flags = Dict(flags)

    @property
    def kernel(self):
        """allow access to the Kernel object, so I look like IPKernelApp"""
        return self.engine.kernel

    def find_url_file(self):
        """Set the url file.

        Here we don't try to actually see if it exists for is valid as that
        is hadled by the connection logic.
        """
        # Find the actual controller key file
        if not self.url_file:
            self.url_file = os.path.join(self.profile_dir.security_dir,
                                         self.url_file_name)

    def load_connector_file(self):
        """load config from a JSON connector file,
        at a *lower* priority than command-line/config files.
        """

        self.log.info("Loading url_file %r", self.url_file)
        config = self.config

        with open(self.url_file) as f:
            num_tries = 0
            max_tries = 5
            d = ""
            while not d:
                try:
                    d = json.loads(f.read())
                except ValueError:
                    if num_tries > max_tries:
                        raise
                    num_tries += 1
                    time.sleep(0.5)

        # allow hand-override of location for disambiguation
        # and ssh-server
        if 'EngineFactory.location' not in config:
            config.EngineFactory.location = d['location']
        if 'EngineFactory.sshserver' not in config:
            config.EngineFactory.sshserver = d.get('ssh')

        location = config.EngineFactory.location

        proto, ip = d['interface'].split('://')
        ip = disambiguate_ip_address(ip, location)
        d['interface'] = '%s://%s' % (proto, ip)

        # DO NOT allow override of basic URLs, serialization, or key
        # JSON file takes top priority there
        config.Session.key = cast_bytes(d['key'])
        config.Session.signature_scheme = d['signature_scheme']

        config.EngineFactory.url = d['interface'] + ':%i' % d['registration']

        config.Session.packer = d['pack']
        config.Session.unpacker = d['unpack']

        self.log.debug("Config changed:")
        self.log.debug("%r", config)
        self.connection_info = d

    def bind_kernel(self, **kwargs):
        """Promote engine to listening kernel, accessible to frontends."""
        if self.kernel_app is not None:
            return

        self.log.info(
            "Opening ports for direct connections as an IPython kernel")

        kernel = self.kernel

        kwargs.setdefault('config', self.config)
        kwargs.setdefault('log', self.log)
        kwargs.setdefault('profile_dir', self.profile_dir)
        kwargs.setdefault('session', self.engine.session)

        app = self.kernel_app = IPKernelApp(**kwargs)

        # allow IPKernelApp.instance():
        IPKernelApp._instance = app

        app.init_connection_file()
        # relevant contents of init_sockets:

        app.shell_port = app._bind_socket(kernel.shell_streams[0],
                                          app.shell_port)
        app.log.debug("shell ROUTER Channel on port: %i", app.shell_port)

        app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)
        app.log.debug("iopub PUB Channel on port: %i", app.iopub_port)

        kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)
        app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)
        app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port)

        # start the heartbeat, and log connection info:

        app.init_heartbeat()

        app.log_connection_info()
        app.connection_dir = self.profile_dir.security_dir
        app.write_connection_file()

    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.config
        # print config
        self.find_url_file()

        # was the url manually specified?
        keys = set(self.config.EngineFactory.keys())
        keys = keys.union(set(self.config.RegistrationFactory.keys()))

        if self.wait_for_url_file and not os.path.exists(self.url_file):
            self.log.warn("url_file %r not found", self.url_file)
            self.log.warn("Waiting up to %.1f seconds for it to arrive.",
                          self.wait_for_url_file)
            tic = time.time()
            while not os.path.exists(self.url_file) and (
                    time.time() - tic < self.wait_for_url_file):
                # wait for url_file to exist, or until time limit
                time.sleep(0.1)

        if os.path.exists(self.url_file):
            self.load_connector_file()
        else:
            self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
            self.exit(1)

        exec_lines = []
        for app in ('IPKernelApp', 'InteractiveShellApp'):
            if '%s.exec_lines' % app in config:
                exec_lines = config[app].exec_lines
                break

        exec_files = []
        for app in ('IPKernelApp', 'InteractiveShellApp'):
            if '%s.exec_files' % app in config:
                exec_files = config[app].exec_files
                break

        config.IPKernelApp.exec_lines = exec_lines
        config.IPKernelApp.exec_files = exec_files

        if self.startup_script:
            exec_files.append(self.startup_script)
        if self.startup_command:
            exec_lines.append(self.startup_command)

        # Create the underlying shell class and Engine
        # shell_class = import_item(self.master_config.Global.shell_class)
        # print self.config
        try:
            self.engine = EngineFactory(
                config=config,
                log=self.log,
                connection_info=self.connection_info,
            )
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)

    def forward_logging(self):
        if self.log_url:
            self.log.info("Forwarding logging to %s", self.log_url)
            context = self.engine.context
            lsock = context.socket(zmq.PUB)
            lsock.connect(self.log_url)
            handler = EnginePUBHandler(self.engine, lsock)
            handler.setLevel(self.log_level)
            self.log.addHandler(handler)

    def init_mpi(self):
        global mpi
        self.mpi = MPI(parent=self)

        mpi_import_statement = self.mpi.init_script
        if mpi_import_statement:
            try:
                self.log.info("Initializing MPI:")
                self.log.info(mpi_import_statement)
                exec(mpi_import_statement, globals())
            except:
                mpi = None
        else:
            mpi = None

    @catch_config_error
    def initialize(self, argv=None):
        super(IPEngineApp, self).initialize(argv)
        self.init_mpi()
        self.init_engine()
        self.forward_logging()

    def start(self):
        self.engine.start()
        try:
            self.engine.loop.start()
        except KeyboardInterrupt:
            self.log.critical("Engine Interrupted, shutting down...\n")
Ejemplo n.º 11
0
class Plot(widgets.DOMWidget):
    """
    Main K3D widget.

    Attributes:
        antialias: `int`:
            Enable antialiasing in WebGL renderer, changes have no effect after displaying.
        height: `int`:
            Height of the Widget in pixels, changes have no effect after displaying.
        background_color: `int`.
            Packed RGB color of the plot background (0xff0000 is red, 0xff is blue), -1 is for transparent.
        camera_auto_fit: `bool`.
            Enable automatic camera setting after adding, removing or changing a plot object.
        grid_auto_fit: `bool`.
            Enable automatic adjustment of the plot grid to contained objects.
        grid_visible: `bool`.
            Enable or disable grid.
        screenshot_scale: `Float`.
            Multipiler to screenshot resolution.
        voxel_paint_color: `int`.
            The (initial) int value to be inserted when editing voxels.
        lighting: `Float`.
            Lighting factor.
        grid: `array_like`.
            6-element tuple specifying the bounds of the plot grid (x0, y0, z0, x1, y1, z1).
        camera: `array_like`.
            9-element list or array specifying camera position.
        camera_no_rotate: `Bool`.
            Lock for camera rotation.
        camera_no_zoom: `Bool`.
            Lock for camera zoom.
        camera_no_pan: `Bool`.
            Lock for camera pan.
        camera_rotate_speed: `Float`.
            Speed of camera rotation.
        camera_zoom_speed: `Float`.
            Speed of camera zoom.
        camera_pan_speed: `Float`.
            Speed of camera pan.
        camera_fov: `Float`.
            Camera Field of View.
        snapshot_include_js: `Bool`.
            If it's true snapshot html is standalone.
        axes: `list`.
            Axes labels for plot.
        time: `list`.
            Time value (used in TimeSeries)
        name: `string`.
            Name of the plot. Used to filenames of snapshot/screenshot etc.
        mode: `str`.
            Mode of K3D viewer.

            Legal values are:

            :`view`: No interaction with objects,

            :`add`: On voxels objects adding mode,

            :`change`: On voxels objects edit mode,

            :`callback`: Handling click_callback and hover_callback on some type of objects,

            :`manipulate`: Enable object transform widget.
        camera_mode: `str`.
            Mode of camera movement.

            Legal values are:

            :`trackball`: orbit around point with dynamic up-vector of camera,

            :`orbit`: orbit around point with fixed up-vector of camera,

            :`fly`: orbit around point with dynamic up-vector of camera, mouse wheel also moves target point.
        manipulate_mode: `str`.
            Mode of manipulate widgets.

            Legal values are:

            :`translate`: Translation widget,

            :`rotate`: Rotation widget,

            :`scale`: Scaling widget.
        auto_rendering: `Bool`.
            State of auto rendering.
        fps: `Float`.
            Fps of animation.
        objects: `list`.
            List of `k3d.objects.Drawable` currently included in the plot, not to be changed directly.
    """

    _view_name = Unicode('PlotView').tag(sync=True)
    _model_name = Unicode('PlotModel').tag(sync=True)
    _view_module = Unicode('k3d').tag(sync=True)
    _model_module = Unicode('k3d').tag(sync=True)

    _view_module_version = Unicode(version).tag(sync=True)
    _model_module_version = Unicode(version).tag(sync=True)
    _backend_version = Unicode(version).tag(sync=True)

    # readonly (specified at creation)
    antialias = Int(min=0, max=5).tag(sync=True)
    height = Int().tag(sync=True)

    # readonly (not to be modified directly)
    object_ids = List().tag(sync=True)

    # read-write
    camera_auto_fit = Bool(True).tag(sync=True)
    auto_rendering = Bool(True).tag(sync=True)
    snapshot_include_js = Bool(True).tag(sync=True)
    lighting = Float().tag(sync=True)
    fps = Float().tag(sync=True)
    grid_auto_fit = Bool(True).tag(sync=True)
    grid_visible = Bool(True).tag(sync=True)
    fps_meter = Bool(True).tag(sync=True)
    menu_visibility = Bool(True).tag(sync=True)
    screenshot_scale = Float().tag(sync=True)
    time = Float().tag(sync=True)
    grid = ListOrArray((-1, -1, -1, 1, 1, 1), minlen=6,
                       maxlen=6).tag(sync=True)
    background_color = Int().tag(sync=True)
    voxel_paint_color = Int().tag(sync=True)
    camera = ListOrArray(minlen=9, maxlen=9, empty_ok=True).tag(sync=True)
    camera_animation = TimeSeries(
        ListOrArray(minlen=9, maxlen=9, empty_ok=True)).tag(sync=True)
    camera_no_rotate = Bool(False).tag(sync=True)
    camera_no_zoom = Bool(False).tag(sync=True)
    camera_no_pan = Bool(False).tag(sync=True)
    camera_rotate_speed = Float().tag(sync=True)
    camera_zoom_speed = Float().tag(sync=True)
    camera_pan_speed = Float().tag(sync=True)
    clipping_planes = ListOrArray(empty_ok=True).tag(sync=True)
    colorbar_object_id = Int(-1).tag(sync=True)
    colorbar_scientific = Bool(False).tag(sync=True)
    rendering_steps = Int(1).tag(sync=True)
    screenshot = Unicode().tag(sync=True)
    snapshot = Unicode().tag(sync=True)
    camera_fov = Float().tag(sync=True)
    name = Unicode(default_value=None, allow_none=True).tag(sync=True)
    axes = List(minlen=3, maxlen=3, default_value=['x', 'y',
                                                   'z']).tag(sync=True)
    axes_helper = Float().tag(sync=True)
    mode = Unicode().tag(sync=True)
    camera_mode = Unicode().tag(sync=True)
    manipulate_mode = Unicode().tag(sync=True)

    objects = []

    def __init__(self,
                 antialias=3,
                 background_color=0xFFFFFF,
                 camera_auto_fit=True,
                 grid_auto_fit=True,
                 grid_visible=True,
                 height=512,
                 voxel_paint_color=0,
                 grid=(-1, -1, -1, 1, 1, 1),
                 screenshot_scale=2.0,
                 lighting=1.5,
                 time=0.0,
                 fps_meter=False,
                 menu_visibility=True,
                 colorbar_object_id=-1,
                 rendering_steps=1,
                 axes=['x', 'y', 'z'],
                 camera_no_rotate=False,
                 camera_no_zoom=False,
                 snapshot_include_js=True,
                 camera_no_pan=False,
                 camera_rotate_speed=1.0,
                 camera_zoom_speed=1.2,
                 camera_pan_speed=0.3,
                 camera_fov=45.0,
                 axes_helper=1.0,
                 name=None,
                 mode='view',
                 camera_mode='trackball',
                 manipulate_mode='translate',
                 auto_rendering=True,
                 fps=25.0,
                 *args,
                 **kwargs):
        super(Plot, self).__init__()

        self.antialias = antialias
        self.camera_auto_fit = camera_auto_fit
        self.grid_auto_fit = grid_auto_fit
        self.fps_meter = fps_meter
        self.fps = fps
        self.grid = grid
        self.grid_visible = grid_visible
        self.background_color = background_color
        self.voxel_paint_color = voxel_paint_color
        self.screenshot_scale = screenshot_scale
        self.height = height
        self.lighting = lighting
        self.time = time
        self.menu_visibility = menu_visibility
        self.colorbar_object_id = colorbar_object_id
        self.rendering_steps = rendering_steps
        self.camera_no_rotate = camera_no_rotate
        self.camera_no_zoom = camera_no_zoom
        self.camera_no_pan = camera_no_pan
        self.camera_rotate_speed = camera_rotate_speed
        self.camera_zoom_speed = camera_zoom_speed
        self.camera_pan_speed = camera_pan_speed
        self.camera_fov = camera_fov
        self.axes = axes
        self.axes_helper = axes_helper
        self.name = name
        self.mode = mode
        self.snapshot_include_js = snapshot_include_js
        self.camera_mode = camera_mode
        self.manipulate_mode = manipulate_mode
        self.auto_rendering = auto_rendering
        self.camera = [2, -3, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]

        self.object_ids = []
        self.objects = []

        self.outputs = []

    def __iadd__(self, objs):
        """Add Drawable to plot."""
        assert isinstance(objs, Drawable)

        for obj in objs:
            if obj.id not in self.object_ids:
                self.object_ids = self.object_ids + [obj.id]
                self.objects.append(obj)

        return self

    def __isub__(self, objs):
        """Remove Drawable from plot."""
        assert isinstance(objs, Drawable)

        for obj in objs:
            self.object_ids = [id_ for id_ in self.object_ids if id_ != obj.id]
            if obj in self.objects:
                self.objects.remove(obj)

        return self

    def display(self, **kwargs):
        """Show plot inside ipywidgets.Output()."""
        output = widgets.Output()

        with output:
            display(self, **kwargs)

        self.outputs.append(output)

        display(output)

    def render(self):
        """Trigger rendering on demand.

        Useful when self.auto_rendering == False."""
        self.send({'msg_type': 'render'})

    def start_auto_play(self):
        """Start animation of plot with objects using TimeSeries."""
        self.send({'msg_type': 'start_auto_play'})

    def stop_auto_play(self):
        """Stop animation of plot with objects using TimeSeries."""
        self.send({'msg_type': 'stop_auto_play'})

    def close(self):
        """Remove plot from all its ipywidgets.Output()-s."""
        for output in self.outputs:
            output.clear_output()

        self.outputs = []

    def camera_reset(self, factor=1.5):
        """Trigger auto-adjustment of camera.

        Useful when self.camera_auto_fit == False."""
        self.send({'msg_type': 'reset_camera', 'factor': factor})

    def get_auto_grid(self):
        d = np.stack([o.get_bounding_box() for o in self.objects])

        return np.dstack(
            [np.min(d[:, 0::2], axis=0),
             np.max(d[:, 1::2], axis=0)]).flatten()

    def get_auto_camera(self, factor=1.5, yaw=25, pitch=15):
        bounds = self.get_auto_grid()
        center = (bounds[::2] + bounds[1::2]) / 2.0
        radius = 0.5 * np.sum(np.abs(bounds[::2] - bounds[1::2])**2)**0.5
        cam_distance = radius * factor / np.sin(
            np.deg2rad(self.camera_fov / 2.0))

        x = np.sin(np.deg2rad(pitch)) * np.cos(np.deg2rad(yaw))
        y = np.sin(np.deg2rad(pitch)) * np.sin(np.deg2rad(yaw))
        z = np.cos(np.deg2rad(pitch))

        if pitch not in [0, 180]:
            up = [0, 0, 1]
        else:
            up = [0, 1, 1]

        return [
            center[0] + x * cam_distance, center[1] + y * cam_distance,
            center[2] + z * cam_distance, center[0], center[1], center[2],
            up[0], up[1], up[2]
        ]

    def fetch_screenshot(self, only_canvas=False):
        """Request creating a PNG screenshot on the JS side and saving it in self.screenshot

        The result is a string of a PNG file in base64 encoding.
        This function requires a round-trip of websocket messages. The result will
        be available after the current cell finishes execution."""
        self.send({'msg_type': 'fetch_screenshot', 'only_canvas': only_canvas})

    def yield_screenshots(self, generator_function):
        """Decorator for a generator function receiving screenshots via yield."""
        @wraps(generator_function)
        def inner():
            generator = generator_function()

            def send_new_value(change):
                try:
                    generator.send(base64.b64decode(change.new))
                except StopIteration:
                    self.unobserve(send_new_value, 'screenshot')

            self.observe(send_new_value, 'screenshot')
            # start the decorated generator
            generator.send(None)

        return inner

    def fetch_snapshot(self, compression_level=9):
        """Request creating a HTML snapshot on the JS side and saving it in self.snapshot

        The result is a string: a HTML document with this plot embedded.
        This function requires a round-trip of websocket messages. The result will
        be available after the current cell finishes execution."""
        self.send({
            'msg_type': 'fetch_snapshot',
            'compression_level': compression_level
        })

    def yield_snapshots(self, generator_function):
        """Decorator for a generator function receiving snapshots via yield."""
        @wraps(generator_function)
        def inner():
            generator = generator_function()

            def send_new_value(change):
                try:
                    generator.send(base64.b64decode(change.new))
                except StopIteration:
                    self.unobserve(send_new_value, 'snapshot')

            self.observe(send_new_value, 'snapshot')
            # start the decorated generator
            generator.send(None)

        return inner

    def get_binary_snapshot_objects(self):
        import msgpack
        from .helpers import to_json

        snapshot = {"objects": [], "chunkList": []}

        for o in self.objects:
            obj = {}
            for k, v in o.traits().items():
                if 'sync' in v.metadata:
                    obj[k] = to_json(k, o[k], o, o['compression_level'])

            snapshot['objects'].append(obj)

        return msgpack.packb(snapshot, use_bin_type=True)

    def get_snapshot_params(self):
        return {
            "cameraAutoFit": self.camera_auto_fit,
            "menuVisibility": self.menu_visibility,
            "gridAutoFit": self.grid_auto_fit,
            "gridVisible": self.grid_visible,
            "grid": self.grid,
            "antialias": self.antialias,
            "screenshotScale": self.screenshot_scale,
            "clearColor": self.background_color,
            "clippingPlanes": self.clipping_planes,
            "lighting": self.lighting,
            "time": self.time,
            "fpsMeter": self.fps_meter,
            "cameraMode": self.camera_mode,
            "colorbarObjectId": self.colorbar_object_id,
            "axes": self.axes,
            "cameraNoRotate": self.camera_no_rotate,
            "cameraNoZoom": self.camera_no_zoom,
            "cameraNoPan": self.camera_no_pan,
            "cameraRotateSpeed": self.camera_rotate_speed,
            "cameraZoomSpeed": self.camera_zoom_speed,
            "cameraPanSpeed": self.camera_pan_speed,
            "name": self.name,
            "camera_fov": self.camera_fov,
            "axesHelper": self.axes_helper,
            "cameraAnimation": self.camera_animation,
            "fps": self.fps
        }

    def get_snapshot(self, compression_level=9, additional_js_code=''):
        """Produce on the Python side a HTML document with the current plot embedded."""
        import os
        import io
        import zlib

        dir_path = os.path.dirname(os.path.realpath(__file__))

        data = self.get_binary_snapshot_objects()
        data = base64.b64encode(zlib.compress(data, compression_level))

        if self.snapshot_include_js:
            f = io.open(os.path.join(dir_path, 'static',
                                     'snapshot_standalone.txt'),
                        mode="r",
                        encoding="utf-8")
            template = f.read()
            f.close()

            f = io.open(os.path.join(dir_path, 'static', 'standalone.js'),
                        mode="r",
                        encoding="utf-8")
            template = template.replace(
                '[K3D_SOURCE]',
                base64.b64encode(
                    zlib.compress(f.read().encode(),
                                  compression_level)).decode("utf-8"))
            f.close()

            f = io.open(os.path.join(dir_path, 'static', 'require.js'),
                        mode="r",
                        encoding="utf-8")
            template = template.replace('[REQUIRE_JS]', f.read())
            f.close()

            f = io.open(os.path.join(dir_path, 'static',
                                     'pako_inflate.min.js'),
                        mode="r",
                        encoding="utf-8")
            template = template.replace('[PAKO_JS]', f.read())
            f.close()
        else:
            f = io.open(os.path.join(dir_path, 'static',
                                     'snapshot_online.txt'),
                        mode="r",
                        encoding="utf-8")
            template = f.read()
            f.close()

            template = template.replace('[VERSION]', self._view_module_version)

        template = template.replace('[DATA]', data.decode("utf-8"))

        params = self.get_snapshot_params()

        template = template.replace('[PARAMS]', json.dumps(params))
        template = template.replace('[CAMERA]', str(self.camera))
        template = template.replace('[ADDITIONAL]', additional_js_code)

        return template
Ejemplo n.º 12
0
class TFMaker(Component):
    name = 'TFMaker'

    vped_list = List(Int,
                     None,
                     allow_none=True,
                     help='List of the vped value for each input '
                     'file').tag(config=True)
    pedestal_path = Unicode(None,
                            allow_none=True,
                            help='Path to the pedestal file (TF requires the '
                            'pedestal to be first subtracted before '
                            'generating').tag(config=True)
    adc_step = Int(8, help='Step in ADC that the TF file will be stored '
                   'in').tag(config=True)
    output_path = Unicode(None,
                          allow_none=True,
                          help='Path to save the TargetCalib pedestal '
                          'file').tag(config=True)
    number_tms = Int(32, help='Number of TARGET modules '
                     'connected').tag(config=True)
    vped_zero = Int(1050, help='VPed value for the pedestal').tag(config=True)
    compress = Bool(False, help='Compress the TF file?').tag(config=True)
    tf_input = Bool(False,
                    help='Create a numpy file containing the input TF '
                    'array before the switch of '
                    'axis').tag(config=True)

    def __init__(self, config, tool, **kwargs):
        """
        Generator of Transfer Function files.

        Parameters
        ----------
        config : traitlets.loader.Config
            Configuration specified by config file or cmdline arguments.
            Used to set traitlet values.
            Set to None if no configuration to pass.
        tool : ctapipe.core.Tool
            Tool executable that is calling this component.
            Passes the correct logger to the component.
            Set to None if no Tool to pass.
        kwargs
        """
        super().__init__(config=config, parent=tool, **kwargs)
        if self.vped_list is None:
            raise ValueError("Please supply vped_list")
        if self.pedestal_path is None:
            raise ValueError("Please specify a pedestal path")
        if self.output_path is None:
            raise ValueError("Please specify an output path to save "
                             "TF file")

        self.ped = PedestalSubtractor(config=config,
                                      tool=tool,
                                      pedestal_path=self.pedestal_path)

        vpeds = np.array(self.vped_list, dtype=np.uint16)
        self.tf_obj = TCTfMaker(vpeds, self.number_tms, self.vped_zero)
        self.current_vped = None

    def add_event(self, event, vped):
        """
        Add an event into the transfer function.

        Parameters
        ----------
        event : container
            A `ctapipe` event container
        vped: int
            The vped of file from which the event comes from
        """
        if self.current_vped != vped:
            self.current_vped = vped
            self.tf_obj.SetVpedIndex(vped)
        telid = 0
        tm = event.meta['tm']
        tmpix = event.meta['tmpix']
        waveforms = event.r0.tel[telid].adc_samples[0]
        first_cell_ids = event.r0.tel[telid].first_cell_ids
        pedsub = np.zeros(waveforms.shape, dtype=np.float32)
        self.ped.apply(event, pedsub)
        self.tf_obj.AddEvent(pedsub, first_cell_ids)

    def save(self):
        """
        Save the pedestal file.
        """
        self.log.info("Saving transfer function to: {}".format(
            self.output_path))
        self.tf_obj.Save(self.output_path, self.adc_step, self.compress)

        if self.tf_input:
            self.save_tf_input()

    def save_tf_input(self):
        tf_input = np.array(self.tf_obj.GetTf())
        vped_vector = np.array(self.tf_obj.GetVpedVector())
        tfinput_path = os.path.splitext(self.output_path)[0] + '_input.npy'
        vped_path = os.path.splitext(self.output_path)[0] + '_vped.npy'
        self.log.info("Saving TF input array to: {}".format(tfinput_path))
        np.save(tfinput_path, tf_input)
        self.log.info("Saving Vped vector to: {}".format(vped_path))
        np.save(vped_path, vped_vector)
Ejemplo n.º 13
0
class ProjView(RegulusDOMWidget):
    _model_name = Unicode('ProjModel').tag(sync=True)
    _view_name = Unicode('ProjView').tag(sync=True)

    data = Instance(klass=DataWidget,
                    allow_none=True).tag(sync=True, **widget_serialization)
    show = List(Int()).tag(sync=True)
    show_graph = Bool(True).tag(sync=True)
    show_pts = Bool(False).tag(sync=True)
    show_inverse = Bool(False).tag(sync=True)
    highlight = Int(-2).tag(sync=True)
    axes = TypedTuple(trait=AxisTraitType()).tag(sync=True,
                                                 **widget_serialization)
    color = Unicode(None, allow_none=True).tag(sync=True)
    color_info = List((None, 0, 1)).tag(sync=True)
    # colors = Any([]).tag(sync=True)
    inverse = Dict(allow_none=True).tag(sync=True)

    def __init__(self, data=None, **kwargs):
        self._inverse_cache = set()
        super().__init__(**kwargs)
        if data is not None:
            self.data = data

    @validate('data')
    def _valid_value(self, proposal):
        data = proposal['value']
        if data is not None and not isinstance(data, DataWidget):
            data = DataWidget(data=data)
        return data

    @observe('color')
    def color_changed(self, _):
        if self.color is None or self.color is not '':
            self.color_info = (None, 0, 1)
        if self.data is not None and self.data.data is not None:
            data = self.data.data
            if self.color in data.values:
                c = data.values[self.color]
                self.color_info = (list(data.values).index(self.color),
                                   c.min(), c.max())
            else:
                print('invalid color', self.color)

    def reset_inverse(self):
        self._inverse_cache.clear()
        self._show({'new': self.show})

    def _send_msg(self, pid, data):
        self.inverse = {pid: data}
        self.inverse = None

    @observe('show')
    def _show(self, change):
        self._update_inverse()

    @observe('show_inverse')
    def _show_inverse(self, change):
        self._update_inverse()

    def _update_inverse(self):
        r = self.data.data
        if self.show_inverse and self.data is not None:
            if not self.data.data.attr.has('inverse_regression'):
                return
            pids = filter(lambda pid: pid not in self._inverse_cache,
                          self.show)
            msg = {}
            t0 = time()
            for node in r.find_nodes(pids):
                curve, std = r.attr['inverse_regression'][node]
                msg[node.id] = curve.reset_index().values.tolist()
                self._inverse_cache.add(node.id)
                if time() - t0 > 1:
                    self.inverse = msg
                    self.inverse = None
                    msg = {}
                    t0 = time()
            if len(msg) > 0:
                self.inverse = msg
                self.inverse = None
Ejemplo n.º 14
0
class PredictionApplication(Application):
    name = Unicode(u'hdnnpy predict')
    description = ('Predict properties for atomic structures using trained'
                   ' HDNNP.')

    verbose = Bool(
        False,
        help='Set verbose mode'
        ).tag(config=True)

    classes = List([PredictionConfig])

    config_file = Path(
        'prediction_config.py',
        help='Load this config file')

    aliases = Dict({
        'log_level': 'Application.log_level',
        })

    flags = Dict({
        'verbose': ({
            'PredictionApplication': {
                'verbose': True,
                },
            }, 'Set verbose mode'),
        'v': ({
            'PredictionApplication': {
                'verbose': True,
                },
            }, 'Set verbose mode'),
        'debug': ({
            'Application': {
                'log_level': 10,
                },
            }, 'Set log level to DEBUG'),
        })

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.dataset_config = None
        self.model_config = None
        self.prediction_config = None

    def initialize(self, argv=None):
        self.parse_command_line(argv)
        self.load_config_file(self.config_file)
        self.prediction_config = PredictionConfig(config=self.config)

        yaml.add_constructor('Path', pyyaml_path_constructor)
        training_result = yaml.load(
            (self.prediction_config.load_dir / 'training_result.yaml').open())
        self.dataset_config = DatasetConfig(**training_result['dataset'])
        self.model_config = ModelConfig(**training_result['model'])

    def start(self):
        pc = self.prediction_config
        shutil.copy(self.config_file, pc.load_dir / self.config_file.name)
        tag_xyz_map, pc.elements = parse_xyz(
            pc.data_file, save=False, verbose=self.verbose)
        datasets = self.construct_datasets(tag_xyz_map)
        datasets = DatasetGenerator(*datasets).all()
        if MPI.rank == 0:
            results = self.predict(datasets)
            self.dump_result(results)

    def construct_datasets(self, tag_xyz_map):
        dc = self.dataset_config
        mc = self.model_config
        pc = self.prediction_config

        preprocesses = []
        for (name, args, kwargs) in dc.preprocesses:
            preprocess = PREPROCESS[name](*args, **kwargs)
            preprocess.load(
                pc.load_dir / 'preprocess' / f'{name}.npz',
                verbose=self.verbose)
            preprocesses.append(preprocess)

        datasets = []
        for pattern in pc.tags:
            for tag in fnmatch.filter(tag_xyz_map, pattern):
                if self.verbose:
                    pprint(f'Construct sub dataset tagged as "{tag}"')
                tagged_xyz = tag_xyz_map.pop(tag)
                structures = AtomicStructure.read_xyz(tagged_xyz)

                # prepare descriptor dataset
                descriptor = DESCRIPTOR_DATASET[dc.descriptor](
                    pc.order, structures, **dc.parameters)
                descriptor.make(verbose=self.verbose)

                # prepare empty property dataset
                property_ = PROPERTY_DATASET[dc.property_](
                    pc.order, structures)

                # construct test dataset from descriptor & property datasets
                dataset = HDNNPDataset(descriptor, property_)
                dataset.construct(
                    all_elements=pc.elements, preprocesses=preprocesses,
                    shuffle=False, verbose=self.verbose)
                datasets.append(dataset)
                dc.n_sample += dataset.total_size
                mc.n_input = dataset.n_input
                mc.n_output = dataset.n_label

        return datasets

    def predict(self, datasets):
        mc = self.model_config
        pc = self.prediction_config
        results = []

        # master model
        master_nnp = MasterNNP(
            pc.elements, mc.n_input, mc.hidden_layers, mc.n_output)
        chainer.serializers.load_npz(
            pc.load_dir / 'master_nnp.npz', master_nnp)

        for dataset in datasets:
            # hdnnp model
            hdnnp = HighDimensionalNNP(
                dataset.elemental_composition,
                mc.n_input, mc.hidden_layers, mc.n_output)
            hdnnp.sync_param_with(master_nnp)

            batch = chainer.dataset.concat_examples(dataset)
            inputs = [batch[f'inputs/{i}'] for i in range(pc.order + 1)]
            with chainer.using_config('train', False), \
                 chainer.using_config('enable_backprop', False):
                predictions = hdnnp.predict(inputs, pc.order)

            result = {
                **{'tag': dataset.tag},
                **{property_: coefficient * prediction.data
                   for property_, coefficient, prediction
                   in zip(dataset.property.properties,
                          dataset.property.coefficients,
                          predictions)},
                }
            results.append(result)
        return results

    def dump_result(self, results):
        pc = self.prediction_config
        result_file = pc.load_dir / f'prediction_result{pc.dump_format}'
        if pc.dump_format == '.npz':
            kv_result = {}
            for result in results:
                tag = result.pop('tag')
                kv_result.update({tag + '/' + key: value
                                  for key, value in result.items()})
            np.savez(result_file, **kv_result)
Ejemplo n.º 15
0
class WebServer(Application):

    port = Integer(8081,
                   help="""The internal port for the Hub process.
        
        This is the internal port of the hub itself. It should never be accessed directly.
        See JupyterHub.port for the public port to use when accessing jupyterhub.
        It is rare that this port should be set except in cases of port conflict.
        """).tag(config=True)
    ip = Unicode('127.0.0.1',
                 help="""The ip address for the Hub process to *bind* to.
        
        By default, the hub listens on localhost only. This address must be accessible from 
        the proxy and user servers. You may need to set this to a public ip or '' for all 
        interfaces if the proxy or user servers are in containers or on a different host.

        See `hub_connect_ip` for cases where the bind and connect address should differ.
        """).tag(config=True)

    config_file = Unicode(
        'config.py',
        help="The config file to load",
    ).tag(config=True)

    tornado_settings = Dict(
        help="Extra settings overrides to pass to the tornado application."
    ).tag(config=True)

    handlers = List()

    async def start(self):
        """Start the whole thing"""
        self.io_loop = loop = IOLoop.current()

        if self.subapp:
            self.subapp.start()
            loop.stop()
            return

    def init_handlers(self):
        h = []
        # set default handlers
        h.extend(handlers.default_handlers)
        h.extend(apihandlers.default_handlers)
        self.handlers.extend(h)

    def init_tornado_application(self):
        properties = None
        """Instantiate the tornado Application object"""
        self.tornado_application = web.Application(self.handlers,
                                                   **self.tornado_settings)

        print("\nWeb Server listening from port ", self.config.WebServer.port)
        self.tornado_application.listen(self.config.WebServer.port)

    @catch_config_error
    async def initialize(self, *args):
        self.load_config_file(self.config_file)
        self.init_handlers()
        self.init_tornado_application()

    async def launch_instance_async(self, argv=None):
        try:
            await self.initialize(argv)
            await self.start()
        except Exception as e:
            self.log.exception("%s", e)
            self.exit(1)

    @classmethod
    def launch_instance(cls, argv=None, **kwargs):
        """Launch an instance of a WebServer Application"""
        self = cls.instance()
        AsyncIOMainLoop().install()
        loop = IOLoop.current()
        loop.add_callback(self.launch_instance_async, argv)
        try:
            loop.start()
        except KeyboardInterrupt:
            print("\nKeyboard Interrupted")
Ejemplo n.º 16
0
class ContentsManager(LoggingConfigurable):
    """Base class for serving files and directories.

    This serves any text or binary file,
    as well as directories,
    with special handling for JSON notebook documents.

    Most APIs take a path argument,
    which is always an API-style unicode path,
    and always refers to a directory.

    - unicode, not url-escaped
    - '/'-separated
    - leading and trailing '/' will be stripped
    - if unspecified, path defaults to '',
      indicating the root path.

    """

    notary = Instance(sign.NotebookNotary)

    def _notary_default(self):
        return sign.NotebookNotary(parent=self)

    hide_globs = List(Unicode(), [
        u'__pycache__',
        '*.pyc',
        '*.pyo',
        '.DS_Store',
        '*.so',
        '*.dylib',
        '*~',
    ],
                      config=True,
                      help="""
        Glob patterns to hide in file and directory listings.
    """)

    untitled_notebook = Unicode(
        "Untitled",
        config=True,
        help="The base name used when creating untitled notebooks.")

    untitled_file = Unicode(
        "untitled",
        config=True,
        help="The base name used when creating untitled files.")

    untitled_directory = Unicode(
        "Untitled Folder",
        config=True,
        help="The base name used when creating untitled directories.")

    pre_save_hook = Any(None,
                        config=True,
                        allow_none=True,
                        help="""Python callable or importstring thereof

        To be called on a contents model prior to save.

        This can be used to process the structure,
        such as removing notebook outputs or other side effects that
        should not be saved.

        It will be called as (all arguments passed by keyword)::

            hook(path=path, model=model, contents_manager=self)

        - model: the model to be saved. Includes file contents.
          Modifying this dict will affect the file that is stored.
        - path: the API path of the save destination
        - contents_manager: this ContentsManager instance
        """)

    def _pre_save_hook_changed(self, name, old, new):
        if new and isinstance(new, string_types):
            self.pre_save_hook = import_item(self.pre_save_hook)
        elif new:
            if not callable(new):
                raise TraitError("pre_save_hook must be callable")

    def run_pre_save_hook(self, model, path, **kwargs):
        """Run the pre-save hook if defined, and log errors"""
        if self.pre_save_hook:
            try:
                self.log.debug("Running pre-save hook on %s", path)
                self.pre_save_hook(model=model,
                                   path=path,
                                   contents_manager=self,
                                   **kwargs)
            except Exception:
                self.log.error("Pre-save hook failed on %s",
                               path,
                               exc_info=True)

    checkpoints_class = Type(Checkpoints, config=True)
    checkpoints = Instance(Checkpoints, config=True)
    checkpoints_kwargs = Dict(config=True)

    def _checkpoints_default(self):
        return self.checkpoints_class(**self.checkpoints_kwargs)

    def _checkpoints_kwargs_default(self):
        return dict(
            parent=self,
            log=self.log,
        )

    # ContentsManager API part 1: methods that must be
    # implemented in subclasses.

    def dir_exists(self, path):
        """Does a directory exist at the given path?

        Like os.path.isdir

        Override this method in subclasses.

        Parameters
        ----------
        path : string
            The path to check

        Returns
        -------
        exists : bool
            Whether the path does indeed exist.
        """
        raise NotImplementedError

    def is_hidden(self, path):
        """Is path a hidden directory or file?

        Parameters
        ----------
        path : string
            The path to check. This is an API path (`/` separated,
            relative to root dir).

        Returns
        -------
        hidden : bool
            Whether the path is hidden.

        """
        raise NotImplementedError

    def file_exists(self, path=''):
        """Does a file exist at the given path?

        Like os.path.isfile

        Override this method in subclasses.

        Parameters
        ----------
        path : string
            The API path of a file to check for.

        Returns
        -------
        exists : bool
            Whether the file exists.
        """
        raise NotImplementedError('must be implemented in a subclass')

    def exists(self, path):
        """Does a file or directory exist at the given path?

        Like os.path.exists

        Parameters
        ----------
        path : string
            The API path of a file or directory to check for.

        Returns
        -------
        exists : bool
            Whether the target exists.
        """
        return self.file_exists(path) or self.dir_exists(path)

    def get(self, path, content=True, type=None, format=None):
        """Get a file or directory model."""
        raise NotImplementedError('must be implemented in a subclass')

    def save(self, model, path):
        """
        Save a file or directory model to path.

        Should return the saved model with no content.  Save implementations
        should call self.run_pre_save_hook(model=model, path=path) prior to
        writing any data.
        """
        raise NotImplementedError('must be implemented in a subclass')

    def delete_file(self, path):
        """Delete the file or directory at path."""
        raise NotImplementedError('must be implemented in a subclass')

    def rename_file(self, old_path, new_path):
        """Rename a file or directory."""
        raise NotImplementedError('must be implemented in a subclass')

    # ContentsManager API part 2: methods that have useable default
    # implementations, but can be overridden in subclasses.

    def delete(self, path):
        """Delete a file/directory and any associated checkpoints."""
        path = path.strip('/')
        if not path:
            raise HTTPError(400, "Can't delete root")
        self.delete_file(path)
        self.checkpoints.delete_all_checkpoints(path)

    def rename(self, old_path, new_path):
        """Rename a file and any checkpoints associated with that file."""
        self.rename_file(old_path, new_path)
        self.checkpoints.rename_all_checkpoints(old_path, new_path)

    def update(self, model, path):
        """Update the file's path

        For use in PATCH requests, to enable renaming a file without
        re-uploading its contents. Only used for renaming at the moment.
        """
        path = path.strip('/')
        new_path = model.get('path', path).strip('/')
        if path != new_path:
            self.rename(path, new_path)
        model = self.get(new_path, content=False)
        return model

    def info_string(self):
        return "Serving contents"

    def get_kernel_path(self, path, model=None):
        """Return the API path for the kernel
        
        KernelManagers can turn this value into a filesystem path,
        or ignore it altogether.

        The default value here will start kernels in the directory of the
        notebook server. FileContentsManager overrides this to use the
        directory containing the notebook.
        """
        return ''

    def increment_filename(self, filename, path='', insert=''):
        """Increment a filename until it is unique.

        Parameters
        ----------
        filename : unicode
            The name of a file, including extension
        path : unicode
            The API path of the target's directory

        Returns
        -------
        name : unicode
            A filename that is unique, based on the input filename.
        """
        path = path.strip('/')
        basename, ext = os.path.splitext(filename)
        for i in itertools.count():
            if i:
                insert_i = '{}{}'.format(insert, i)
            else:
                insert_i = ''
            name = u'{basename}{insert}{ext}'.format(basename=basename,
                                                     insert=insert_i,
                                                     ext=ext)
            if not self.exists(u'{}/{}'.format(path, name)):
                break
        return name

    def validate_notebook_model(self, model):
        """Add failed-validation message to model"""
        try:
            validate(model['content'])
        except ValidationError as e:
            model['message'] = u'Notebook Validation failed: {}:\n{}'.format(
                e.message,
                json.dumps(e.instance,
                           indent=1,
                           default=lambda obj: '<UNKNOWN>'),
            )
        return model

    def new_untitled(self, path='', type='', ext=''):
        """Create a new untitled file or directory in path
        
        path must be a directory
        
        File extension can be specified.
        
        Use `new` to create files with a fully specified path (including filename).
        """
        path = path.strip('/')
        if not self.dir_exists(path):
            raise HTTPError(404, 'No such directory: %s' % path)

        model = {}
        if type:
            model['type'] = type

        if ext == '.ipynb':
            model.setdefault('type', 'notebook')
        else:
            model.setdefault('type', 'file')

        insert = ''
        if model['type'] == 'directory':
            untitled = self.untitled_directory
            insert = ' '
        elif model['type'] == 'notebook':
            untitled = self.untitled_notebook
            ext = '.ipynb'
        elif model['type'] == 'file':
            untitled = self.untitled_file
        else:
            raise HTTPError(400, "Unexpected model type: %r" % model['type'])

        name = self.increment_filename(untitled + ext, path, insert=insert)
        path = u'{0}/{1}'.format(path, name)
        return self.new(model, path)

    def new(self, model=None, path=''):
        """Create a new file or directory and return its model with no content.
        
        To create a new untitled entity in a directory, use `new_untitled`.
        """
        path = path.strip('/')
        if model is None:
            model = {}

        if path.endswith('.ipynb'):
            model.setdefault('type', 'notebook')
        else:
            model.setdefault('type', 'file')

        # no content, not a directory, so fill out new-file model
        if 'content' not in model and model['type'] != 'directory':
            if model['type'] == 'notebook':
                model['content'] = new_notebook()
                model['format'] = 'json'
            else:
                model['content'] = ''
                model['type'] = 'file'
                model['format'] = 'text'

        model = self.save(model, path)
        return model

    def copy(self, from_path, to_path=None):
        """Copy an existing file and return its new model.

        If to_path not specified, it will be the parent directory of from_path.
        If to_path is a directory, filename will increment `from_path-Copy#.ext`.

        from_path must be a full path to a file.
        """
        path = from_path.strip('/')
        if to_path is not None:
            to_path = to_path.strip('/')

        if '/' in path:
            from_dir, from_name = path.rsplit('/', 1)
        else:
            from_dir = ''
            from_name = path

        model = self.get(path)
        model.pop('path', None)
        model.pop('name', None)
        if model['type'] == 'directory':
            raise HTTPError(400, "Can't copy directories")

        if to_path is None:
            to_path = from_dir
        if self.dir_exists(to_path):
            name = copy_pat.sub(u'.', from_name)
            to_name = self.increment_filename(name, to_path, insert='-Copy')
            to_path = u'{0}/{1}'.format(to_path, to_name)

        model = self.save(model, to_path)
        return model

    def log_info(self):
        self.log.info(self.info_string())

    def trust_notebook(self, path):
        """Explicitly trust a notebook

        Parameters
        ----------
        path : string
            The path of a notebook
        """
        model = self.get(path)
        nb = model['content']
        self.log.warn("Trusting notebook %s", path)
        self.notary.mark_cells(nb, True)
        self.save(model, path)

    def check_and_sign(self, nb, path=''):
        """Check for trusted cells, and sign the notebook.

        Called as a part of saving notebooks.

        Parameters
        ----------
        nb : dict
            The notebook dict
        path : string
            The notebook's path (for logging)
        """
        if self.notary.check_cells(nb):
            self.notary.sign(nb)
        else:
            self.log.warn("Saving untrusted notebook %s", path)

    def mark_trusted_cells(self, nb, path=''):
        """Mark cells as trusted if the notebook signature matches.

        Called as a part of loading notebooks.

        Parameters
        ----------
        nb : dict
            The notebook object (in current nbformat)
        path : string
            The notebook's path (for logging)
        """
        trusted = self.notary.check_signature(nb)
        if not trusted:
            self.log.warn("Notebook %s is not trusted", path)
        self.notary.mark_cells(nb, trusted)

    def should_list(self, name):
        """Should this file/directory name be displayed in a listing?"""
        return not any(fnmatch(name, glob) for glob in self.hide_globs)

    # Part 3: Checkpoints API
    def create_checkpoint(self, path):
        """Create a checkpoint."""
        return self.checkpoints.create_checkpoint(self, path)

    def restore_checkpoint(self, checkpoint_id, path):
        """
        Restore a checkpoint.
        """
        self.checkpoints.restore_checkpoint(self, checkpoint_id, path)

    def list_checkpoints(self, path):
        return self.checkpoints.list_checkpoints(path)

    def delete_checkpoint(self, checkpoint_id, path):
        return self.checkpoints.delete_checkpoint(checkpoint_id, path)
Ejemplo n.º 17
0
class DisplayIntegrator(Tool):
    name = "ctapipe-display-integration"
    description = __doc__

    event_index = Int(0, help="Event index to view.").tag(config=True)
    use_event_id = Bool(
        False,
        help=
        "event_index will obtain an event using event_id instead of index.",
    ).tag(config=True)
    telescope = Int(
        None,
        allow_none=True,
        help="Telescope to view. Set to None to display the first"
        "telescope with data.",
    ).tag(config=True)
    channel = Enum([0, 1], 0, help="Channel to view").tag(config=True)

    extractor_product = traits.enum_trait(ImageExtractor,
                                          default="NeighborPeakWindowSum")

    aliases = Dict(
        dict(
            f="EventSource.input_url",
            max_events="EventSource.max_events",
            extractor="DisplayIntegrator.extractor_product",
            E="DisplayIntegrator.event_index",
            T="DisplayIntegrator.telescope",
            C="DisplayIntegrator.channel",
        ))
    flags = Dict(
        dict(id=(
            {
                "DisplayDL1Calib": {
                    "use_event_index": True
                }
            },
            "event_index will obtain an event using event_id instead of index.",
        )))
    classes = List([EventSource] + traits.classes_with_traits(ImageExtractor))

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        # make sure gzip files are seekable
        self.config.SimTelEventSource.back_seekable = True
        self.eventseeker = None
        self.extractor = None
        self.calibrator = None

    def setup(self):
        self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]"

        event_source = self.add_component(EventSource.from_config(parent=self))
        self.eventseeker = self.add_component(
            EventSeeker(event_source, parent=self))
        self.extractor = self.add_component(
            ImageExtractor.from_name(self.extractor_product,
                                     parent=self,
                                     subarray=event_source.subarray))
        self.calibrate = self.add_component(
            CameraCalibrator(parent=self,
                             image_extractor=self.extractor,
                             subarray=event_source.subarray))

    def start(self):
        event_num = self.event_index
        if self.use_event_id:
            event_num = str(event_num)
        event = self.eventseeker[event_num]

        # Calibrate
        self.calibrate(event)

        # Select telescope
        tels = list(event.r0.tels_with_data)
        telid = self.telescope
        if telid is None:
            telid = tels[0]
        if telid not in tels:
            self.log.error("[event] please specify one of the following "
                           "telescopes for this event: {}".format(tels))
            exit()

        extractor_name = self.extractor.__class__.__name__

        plot(event, telid, self.channel, extractor_name)

    def finish(self):
        pass
Ejemplo n.º 18
0
class LeToRIRFusionFeatureExtractor(LeToRFeatureExtractor):
    """
    extract the IR fusion features
    """
    feature_name_pre = Unicode('IRFusion')
    l_text_fields = List(Unicode,
                         default_value=TARGET_TEXT_FIELDS).tag(config=True)
    l_model = List(Unicode,
                   default_value=['lm_dir', 'bm25', 'coordinate',
                                  'tf_idf']).tag(config=True)
    corpus_stat_pre = Unicode(help="the file pre of corpus stats").tag(
        config=True)

    def __init__(self, **kwargs):
        super(LeToRIRFusionFeatureExtractor, self).__init__(**kwargs)
        self.s_model = set(self.l_model)
        self.h_field_h_df = dict()
        if self.corpus_stat_pre:
            l_field_h_df, self.h_corpus_stat = load_corpus_stat(
                self.corpus_stat_pre, self.l_text_fields)
            self.h_field_h_df = dict(l_field_h_df)
            for field in self.l_text_fields:
                assert field in self.h_corpus_stat
                assert field in self.h_field_h_df

    def set_external_info(self, external_info):
        super(LeToRIRFusionFeatureExtractor,
              self).set_external_info(external_info)
        self.h_field_h_df = external_info.h_field_h_df
        self.h_corpus_stat = external_info.h_corpus_stat
        return

    def extract_for_text(self, query, docno, h_q_info, h_doc_info):
        h_feature = {}
        # logging.info('extracting IR fusion for q [%s], doc [%s]', query, docno)
        # logging.info('q_info %s', json.dumps(h_q_info))
        # logging.info('doc_info %s', json.dumps(h_doc_info))

        h_tf = text2lm(query.lower())
        # title_ts = None
        for field in self.l_text_fields:
            total_df = self.h_corpus_stat[field]['total_df']
            avg_doc_len = self.h_corpus_stat[field]['average_len']
            h_doc_df = self.h_field_h_df[field]
            h_doc_tf = {}
            if field in h_doc_info:
                h_doc_tf = text2lm(h_doc_info[field].lower(), clean=True)

            term_stat = TermStat()
            term_stat.set_from_raw(h_tf, h_doc_tf, h_doc_df, total_df,
                                   avg_doc_len)
            # if field == 'title':
            #     title_ts = term_stat
            l_sim_score = term_stat.mul_scores()
            for sim, score in l_sim_score:
                if sim in self.s_model:
                    feature_name = self.feature_name_pre + sim.title(
                    ) + field.title()
                    h_feature[feature_name] = score
        #
        # for feature, score in h_feature.items():
        #     if score != h_old_feature[feature]:
        #         logging.warn('ltr feature value different')
        #         logging.warn('old feature: %s', json.dumps(h_old_feature))
        #         logging.warn('new feature: %s', json.dumps(h_feature))
        #
        #         logging.warn('old ts: %s', title_old_ts.pretty_print())
        #         logging.warn('new ts: %s', title_ts.pretty_print())
        #         logging.warn('query: %s, h_tf: %s', query, json.dumps(h_tf))
        #         break

        return h_feature

    def extract_doc_feature(self, docno, h_doc_info):
        h_feature = {}
        if 'is_wiki' in self.s_model:
            score = 0
            if 'enwp' in docno:
                score = 1
            h_feature[self.feature_name_pre + 'IsWiki'] = score
        return h_feature

    def extract(self, qid, docno, h_q_info, h_doc_info):
        query = h_q_info['query']
        h_feature = self.extract_for_text(query, docno, h_q_info, h_doc_info)
        h_feature.update(self.extract_doc_feature(docno, h_q_info))
        return h_feature
Ejemplo n.º 19
0
class Map(DOMWidget, InteractMixin):
    _view_name = Unicode('LeafletMapView').tag(sync=True)
    _model_name = Unicode('LeafletMapModel').tag(sync=True)
    _view_module = Unicode('jupyter-leaflet').tag(sync=True)
    _model_module = Unicode('jupyter-leaflet').tag(sync=True)

    _view_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)
    _model_module_version = Unicode(EXTENSION_VERSION).tag(sync=True)

    # URL of the window where the map is displayed
    window_url = Unicode(read_only=True).tag(sync=True)

    # Map options
    center = List(def_loc).tag(sync=True, o=True)
    zoom_start = CFloat(12).tag(sync=True, o=True)
    zoom = CFloat(12).tag(sync=True, o=True)
    max_zoom = CFloat(18).tag(sync=True, o=True)
    min_zoom = CFloat(1).tag(sync=True, o=True)
    zoom_delta = CFloat(1).tag(sync=True, o=True)
    zoom_snap = CFloat(1).tag(sync=True, o=True)
    interpolation = Unicode('bilinear').tag(sync=True, o=True)
    crs = Dict(default_value=projections.EPSG3857).tag(sync=True)

    # Specification of the basemap
    basemap = Union(
        (Dict(), Instance(TileLayer)),
        default_value=dict(
            url='https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
            max_zoom=19,
            attribution=
            'Map data (c) <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'
        ))
    modisdate = Unicode('yesterday').tag(sync=True)

    # Interaction options
    dragging = Bool(True).tag(sync=True, o=True)
    touch_zoom = Bool(True).tag(sync=True, o=True)
    scroll_wheel_zoom = Bool(False).tag(sync=True, o=True)
    double_click_zoom = Bool(True).tag(sync=True, o=True)
    box_zoom = Bool(True).tag(sync=True, o=True)
    tap = Bool(True).tag(sync=True, o=True)
    tap_tolerance = Int(15).tag(sync=True, o=True)
    world_copy_jump = Bool(False).tag(sync=True, o=True)
    close_popup_on_click = Bool(True).tag(sync=True, o=True)
    bounce_at_zoom_limits = Bool(True).tag(sync=True, o=True)
    keyboard = Bool(True).tag(sync=True, o=True)
    keyboard_pan_offset = Int(80).tag(sync=True, o=True)
    keyboard_zoom_offset = Int(1).tag(sync=True, o=True)
    inertia = Bool(True).tag(sync=True, o=True)
    inertia_deceleration = Int(3000).tag(sync=True, o=True)
    inertia_max_speed = Int(1500).tag(sync=True, o=True)
    # inertia_threshold = Int(?, o=True).tag(sync=True)
    # fade_animation = Bool(?).tag(sync=True, o=True)
    # zoom_animation = Bool(?).tag(sync=True, o=True)
    zoom_animation_threshold = Int(4).tag(sync=True, o=True)
    # marker_zoom_animation = Bool(?).tag(sync=True, o=True)
    fullscreen = Bool(False).tag(sync=True, o=True)

    options = List(trait=Unicode()).tag(sync=True)

    style = InstanceDict(MapStyle).tag(sync=True, **widget_serialization)
    default_style = InstanceDict(MapStyle).tag(sync=True,
                                               **widget_serialization)
    dragging_style = InstanceDict(MapStyle).tag(sync=True,
                                                **widget_serialization)

    zoom_control = Bool(True)
    attribution_control = Bool(True)

    @default('dragging_style')
    def _default_dragging_style(self):
        return {'cursor': 'move'}

    @default('options')
    def _default_options(self):
        return [name for name in self.traits(o=True)]

    south = Float(def_loc[0], read_only=True).tag(sync=True)
    north = Float(def_loc[0], read_only=True).tag(sync=True)
    east = Float(def_loc[1], read_only=True).tag(sync=True)
    west = Float(def_loc[1], read_only=True).tag(sync=True)

    bottom = Int(0, read_only=True).tag(sync=True)
    top = Int(9007199254740991, read_only=True).tag(sync=True)
    right = Int(0, read_only=True).tag(sync=True)
    left = Int(9007199254740991, read_only=True).tag(sync=True)

    layers = Tuple().tag(trait=Instance(Layer),
                         sync=True,
                         **widget_serialization)

    @default('layers')
    def _default_layers(self):
        basemap = self.basemap if isinstance(self.basemap,
                                             TileLayer) else basemap_to_tiles(
                                                 self.basemap, self.modisdate)

        basemap.base = True

        return (basemap, )

    bounds = Tuple(read_only=True)
    bounds_polygon = Tuple(read_only=True)
    pixel_bounds = Tuple(read_only=True)

    @observe('south', 'north', 'east', 'west')
    def _observe_bounds(self, change):
        self.set_trait('bounds',
                       ((self.south, self.west), (self.north, self.east)))
        self.set_trait('bounds_polygon',
                       ((self.north, self.west), (self.north, self.east),
                        (self.south, self.east), (self.south, self.west)))

    @observe('bottom', 'top', 'right', 'left')
    def _observe_pixel_bounds(self, change):
        self.set_trait('pixel_bounds',
                       ((self.left, self.top), (self.right, self.bottom)))

    def __init__(self, **kwargs):
        self.zoom_control_instance = None
        self.attribution_control_instance = None

        super(Map, self).__init__(**kwargs)
        self.on_msg(self._handle_leaflet_event)

        if self.zoom_control:
            self.zoom_control_instance = ZoomControl()
            self.add_control(self.zoom_control_instance)

        if self.attribution_control:
            self.attribution_control_instance = AttributionControl(
                position='bottomright')
            self.add_control(self.attribution_control_instance)

    @observe('zoom_control')
    def observe_zoom_control(self, change):
        if change['new']:
            self.zoom_control_instance = ZoomControl()
            self.add_control(self.zoom_control_instance)
        else:
            if self.zoom_control_instance is not None and self.zoom_control_instance in self.controls:
                self.remove_control(self.zoom_control_instance)

    @observe('attribution_control')
    def observe_attribution_control(self, change):
        if change['new']:
            self.attribution_control_instance = AttributionControl(
                position='bottomright')
            self.add_control(self.attribution_control_instance)
        else:
            if self.attribution_control_instance is not None and self.attribution_control_instance in self.controls:
                self.remove_control(self.attribution_control_instance)

    _layer_ids = List()

    @validate('layers')
    def _validate_layers(self, proposal):
        '''Validate layers list.

        Makes sure only one instance of any given layer can exist in the
        layers list.
        '''
        self._layer_ids = [layer.model_id for layer in proposal.value]
        if len(set(self._layer_ids)) != len(self._layer_ids):
            raise LayerException(
                'duplicate layer detected, only use each layer once')
        return proposal.value

    def add_layer(self, layer):
        if isinstance(layer, dict):
            layer = basemap_to_tiles(layer)
        if layer.model_id in self._layer_ids:
            raise LayerException('layer already on map: %r' % layer)
        self.layers = tuple([layer for layer in self.layers] + [layer])

    def remove_layer(self, rm_layer):
        if rm_layer.model_id not in self._layer_ids:
            raise LayerException('layer not on map: %r' % rm_layer)
        self.layers = tuple([
            layer for layer in self.layers
            if layer.model_id != rm_layer.model_id
        ])

    def substitute_layer(self, old, new):
        if isinstance(new, dict):
            new = basemap_to_tiles(new)
        if old.model_id not in self._layer_ids:
            raise LayerException(
                'Could not substitute layer: layer not on map.')
        self.layers = tuple([
            new if layer.model_id == old.model_id else layer
            for layer in self.layers
        ])

    def clear_layers(self):
        self.layers = ()

    controls = Tuple().tag(trait=Instance(Control),
                           sync=True,
                           **widget_serialization)
    _control_ids = List()

    @validate('controls')
    def _validate_controls(self, proposal):
        '''Validate controls list.

        Makes sure only one instance of any given layer can exist in the
        controls list.
        '''
        self._control_ids = [c.model_id for c in proposal.value]
        if len(set(self._control_ids)) != len(self._control_ids):
            raise ControlException(
                'duplicate control detected, only use each control once')
        return proposal.value

    def add_control(self, control):
        if control.model_id in self._control_ids:
            raise ControlException('control already on map: %r' % control)
        self.controls = tuple([c for c in self.controls] + [control])

    def remove_control(self, control):
        if control.model_id not in self._control_ids:
            raise ControlException('control not on map: %r' % control)
        self.controls = tuple(
            [c for c in self.controls if c.model_id != control.model_id])

    def clear_controls(self):
        self.controls = ()

    def save(self, outfile, **kwargs):
        """Save the Map to an .html file.

        Parameters
        ----------
        outfile: str or file-like object
            The file to write the HTML output to.
        kwargs: keyword-arguments
            Extra parameters to pass to the ipywidgets.embed.embed_minimal_html function.
        """
        embed_minimal_html(outfile, views=[self], **kwargs)

    def __iadd__(self, item):
        if isinstance(item, Layer):
            self.add_layer(item)
        elif isinstance(item, Control):
            self.add_control(item)
        return self

    def __isub__(self, item):
        if isinstance(item, Layer):
            self.remove_layer(item)
        elif isinstance(item, Control):
            self.remove_control(item)
        return self

    def __add__(self, item):
        if isinstance(item, Layer):
            self.add_layer(item)
        elif isinstance(item, Control):
            self.add_control(item)
        return self

    # Event handling
    _interaction_callbacks = Instance(CallbackDispatcher, ())

    def _handle_leaflet_event(self, _, content, buffers):
        if content.get('event', '') == 'interaction':
            self._interaction_callbacks(**content)

    def on_interaction(self, callback, remove=False):
        self._interaction_callbacks.register_callback(callback, remove=remove)
Ejemplo n.º 20
0
class GenerateApp(DsCreate):

    name = u'generate'
    description = u'''
    Splits an nbgrader assignment into student facing and teacher facing files
    and uses the arguments to determine which sub application should be activated.

    **Behavior:**

    GenerateApp uses three major variables.

    1. ``pipeline_steps``
        * This variable is a list containing the converters and controllers that are applied to the repository.
    2. ``branches``
        * This variable is a list containing the name of git branches and is used by CheckoutControllers (included in the ``pipeline_steps`` list) to move sequentially across the branches.
        * *It is worth noting that the ``pipeline_steps`` list cannot contain more CheckoutControllers than the length of ``branches``.
    
    This app uses nbgrader's preprocessors to create student facing and and teacher facing versions for the README markdown files. 
    The curriculum notebook is saved to each branch. 
    '''
    flags = generate_flags
    aliases = generate_aliases

    edit_branch = Unicode('master',
                          help="""
                          Sets the name of the git branch used for curriculum development.
                          Default: 'curriculum'
                          """).tag(config=True)

    pipeline_steps = List([
        CollectCurriculum,
        ReleaseConverter,
        Commit,
        Push,
        Checkout,
        BaseConverter,
        SourceConverter,
        Commit,
        Push,
        CheckoutEditBranch,
    ]).tag(config=True)

    branches = List(['master', 'solution'],
                    help="""
                    Sets the branches used for the notebook  split.
                    Default: ['master', 'solution']
                    """).tag(config=True)

    def start(self) -> None:
        """
        Activates the application.

        * Adds the name of the edit branch to the application configuration object.
        * Configures the DsPipeline object
        * Adds the branches to the controller objects
        * Initializes a DsPipeline
        * Activates thee pipeline
        """
        super().start()

        c = Config()
        c.edit_branch = self.edit_branch
        c.DsPipeline.steps = self.pipeline_steps
        c.BaseController.branches = self.branches
        self.config.merge(c)
        pipeline = DsPipeline(config=self.config)
        pipeline.start()
Ejemplo n.º 21
0
class Viewer(widgets.DOMWidget):
    """
    Generic object for viewing and labeling Candidate objects in their rendered Contexts.
    """
    _view_name = Unicode('ViewerView').tag(sync=True)
    _view_module = Unicode('viewer').tag(sync=True)
    cids = List().tag(sync=True)
    html = Unicode('<h3>Error!</h3>').tag(sync=True)
    _labels_serialized = Unicode().tag(sync=True)
    _selected_cid = Int().tag(sync=True)

    def __init__(self,
                 candidates,
                 session,
                 gold=[],
                 n_per_page=3,
                 height=225,
                 annotator_name=None):
        """
        Initializes a Viewer.

        The Viewer uses the keyword argument annotator_name to define a AnnotatorLabelKey with that name.

        :param candidates: A Python container of Candidates (e.g., not a CandidateSet, but candidate_set.candidates)
        :param session: The SnorkelSession for the database backend
        :param gold: Optional, Python container of Candidates that are know to have positive labels
        :param n_per_page: Optional, number of Contexts to display per page
        :param height: Optional, the height in pixels of the Viewer
        :param annotator_name: Name of the human using the Viewer, for saving their work. Defaults to system username.
        """
        super(Viewer, self).__init__()
        self.session = session

        # By default, use the username as annotator name
        name = annotator_name if annotator_name is not None else getpass.getuser(
        )

        # Sets up the AnnotationKey to use
        self.annotator = self.session.query(GoldLabelKey).filter(
            GoldLabelKey.name == name).first()
        if self.annotator is None:
            self.annotator = GoldLabelKey(name=name)
            session.add(self.annotator)
            session.commit()

        # Viewer display configs
        self.n_per_page = n_per_page
        self.height = height

        # Note that the candidates are not necessarily commited to the DB, so they *may not have* non-null ids
        # Hence, we index by their position in this list
        # We get the sorted candidates and all contexts required, either from unary or binary candidates
        self.gold = list(gold)
        self.candidates = sorted(list(candidates),
                                 key=lambda c: c[0].char_start)
        self.contexts = list(
            set(c[0].get_parent() for c in self.candidates + self.gold))

        # If committed, sort contexts by id
        try:
            self.contexts = sorted(self.contexts, key=lambda c: c.id)
        except:
            pass

        # Loads existing annotations
        self.annotations = [None] * len(self.candidates)
        self.annotations_stable = [None] * len(self.candidates)
        init_labels_serialized = []
        for i, candidate in enumerate(self.candidates):

            # First look for the annotation in the primary annotations table
            existing_annotation = self.session.query(GoldLabel) \
                .filter(GoldLabel.key == self.annotator) \
                .filter(GoldLabel.candidate == candidate) \
                .first()
            if existing_annotation is not None:
                self.annotations[i] = existing_annotation
                if existing_annotation.value == 1:
                    value_string = 'true'
                elif existing_annotation.value == -1:
                    value_string = 'false'
                else:
                    raise ValueError(
                        str(existing_annotation) +
                        ' has value not in {1, -1}, which Viewer does not support.'
                    )
                init_labels_serialized.append(str(i) + '~~' + value_string)

                # If the annotator label is in the main table, also get its stable version
                context_stable_ids = '~~'.join(
                    [c.stable_id for c in candidate.get_contexts()])
                existing_annotation_stable = self.session.query(StableLabel) \
                                                 .filter(StableLabel.context_stable_ids == context_stable_ids)\
                                                 .filter(StableLabel.annotator_name == name).one_or_none()

                # If stable version is not available, create it here
                # NOTE: This is for versioning issues, should be removed?
                if existing_annotation_stable is None:
                    context_stable_ids = '~~'.join(
                        [c.stable_id for c in candidate.get_contexts()])
                    existing_annotation_stable = StableLabel(context_stable_ids=context_stable_ids,\
                                                             annotator_name=self.annotator.name,\
                                                             split=candidate.split,\
                                                             value=existing_annotation.value)
                    self.session.add(existing_annotation_stable)
                    self.session.commit()

                self.annotations_stable[i] = existing_annotation_stable

        self._labels_serialized = ','.join(init_labels_serialized)

        # Configures message handler
        self.on_msg(self.handle_label_event)

        # display js, construct html and pass on to widget model
        self.render()

    def _tag_span(self, html, cids, gold=False):
        """
        Create the span around a segment of the context associated with one or more candidates / gold annotations
        """
        classes = ['candidate'] if len(cids) > 0 else []
        classes += ['gold-annotation'] if gold else []
        classes += list(map(str, cids))

        # Scrub for non-ascii characters; replace with ?
        return u'<span class="{classes}">{html}</span>'.format(
            classes=' '.join(classes), html=html)

    def _tag_context(self, context, candidates, gold):
        """Given the raw context, tag the spans using the generic _tag_span method"""
        raise NotImplementedError()

    def render(self):
        """Renders viewer pane"""
        cids = []

        # Iterate over pages of contexts
        pid = 0
        pages = []
        N = len(self.contexts)
        for i in range(0, N, self.n_per_page):
            page_cids = []
            lis = []
            for j in range(i, min(N, i + self.n_per_page)):
                context = self.contexts[j]

                # Get the candidates in this context
                candidates = [
                    c for c in self.candidates if c[0].get_parent() == context
                ]
                gold = [g for g in self.gold if g.context_id == context.id]

                # Construct the <li> and page view elements
                li_data = self._tag_context(context, candidates, gold)
                lis.append(LI_HTML.format(data=li_data, context_id=context.id))
                page_cids.append(
                    [self.candidates.index(c) for c in candidates])

            # Assemble the page...
            pages.append(
                PAGE_HTML.format(
                    pid=pid,
                    data=''.join(lis),
                    etc=' style="display: block;"' if i == 0 else ''))
            cids.append(page_cids)
            pid += 1

        # Render in primary Viewer template
        self.cids = cids
        self.html = open(HOME + '/viewer/viewer.html').read() % (
            self.height, ''.join(pages))
        display(Javascript(open(HOME + '/viewer/viewer.js').read()))

    def _get_labels(self):
        """
        De-serialize labels from Javascript widget, map to internal candidate id, and return as list of tuples
        """
        LABEL_MAP = {'true': 1, 'false': -1}
        labels = [
            x.split('~~') for x in self._labels_serialized.split(',')
            if len(x) > 0
        ]
        vals = [(int(cid), LABEL_MAP.get(l, 0)) for cid, l in labels]
        return vals

    def handle_label_event(self, _, content, buffers):
        """
        Handles label event by persisting new label
        """
        if content.get('event', '') == 'set_label':
            cid = content.get('cid', None)
            value = content.get('value', None)
            if value is True:
                value = 1
            elif value is False:
                value = -1
            else:
                raise ValueError('Unexpected label returned from widget: ' +
                                 str(value) +
                                 '. Expected values are True and False.')

            # If label already exists, just update value (in both AnnotatorLabel and StableLabel)
            if self.annotations[cid] is not None:
                if self.annotations[cid].value != value:
                    self.annotations[cid].value = value
                    self.annotations_stable[cid].value = value
                    self.session.commit()

            # Otherwise, create a AnnotatorLabel *and a StableLabel*
            else:
                candidate = self.candidates[cid]

                # Create AnnotatorLabel
                self.annotations[cid] = GoldLabel(key=self.annotator,
                                                  candidate=candidate,
                                                  value=value)
                self.session.add(self.annotations[cid])

                # Create StableLabel
                context_stable_ids = '~~'.join(
                    [c.stable_id for c in candidate.get_contexts()])
                self.annotations_stable[cid] = StableLabel(context_stable_ids=context_stable_ids,\
                                                           annotator_name=self.annotator.name,\
                                                           value=value,\
                                                           split=candidate.split)
                self.session.add(self.annotations_stable[cid])
                self.session.commit()

        elif content.get('event', '') == 'delete_label':
            cid = content.get('cid', None)
            self.session.delete(self.annotations[cid])
            self.annotations[cid] = None
            self.session.delete(self.annotations_stable[cid])
            self.annotations_stable[cid] = None
            self.session.commit()

    def get_selected(self):
        return self.candidates[self._selected_cid]
Ejemplo n.º 22
0
class LoadBalancedView(View):
    """An load-balancing View that only executes via the Task scheduler.

    Load-balanced views can be created with the client's `view` method:

    >>> v = client.load_balanced_view()

    or targets can be specified, to restrict the potential destinations:

    >>> v = client.load_balanced_view([1,3])

    which would restrict loadbalancing to between engines 1 and 3.

    """

    follow=Any()
    after=Any()
    timeout=CFloat()
    retries = Integer(0)

    _task_scheme = Any()
    _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])

    def __init__(self, client=None, socket=None, **flags):
        super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
        self._task_scheme=client._task_scheme

    def _validate_dependency(self, dep):
        """validate a dependency.

        For use in `set_flags`.
        """
        if dep is None or isinstance(dep, string_types + (AsyncResult, Dependency)):
            return True
        elif isinstance(dep, (list,set, tuple)):
            for d in dep:
                if not isinstance(d, string_types + (AsyncResult,)):
                    return False
        elif isinstance(dep, dict):
            if set(dep.keys()) != set(Dependency().as_dict().keys()):
                return False
            if not isinstance(dep['msg_ids'], list):
                return False
            for d in dep['msg_ids']:
                if not isinstance(d, string_types):
                    return False
        else:
            return False

        return True

    def _render_dependency(self, dep):
        """helper for building jsonable dependencies from various input forms."""
        if isinstance(dep, Dependency):
            return dep.as_dict()
        elif isinstance(dep, AsyncResult):
            return dep.msg_ids
        elif dep is None:
            return []
        else:
            # pass to Dependency constructor
            return list(Dependency(dep))

    def set_flags(self, **kwargs):
        """set my attribute flags by keyword.

        A View is a wrapper for the Client's apply method, but with attributes
        that specify keyword arguments, those attributes can be set by keyword
        argument with this method.

        Parameters
        ----------

        block : bool
            whether to wait for results
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.

        after : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a time-based dependency.
            This job will only be run *after* the dependencies
            have been met.

        follow : Dependency or collection of msg_ids
            Only for load-balanced execution (targets=None)
            Specify a list of msg_ids as a location-based dependency.
            This job will only be run on an engine where this dependency
            is met.

        timeout : float/int or None
            Only for load-balanced execution (targets=None)
            Specify an amount of time (in seconds) for the scheduler to
            wait for dependencies to be met before failing with a
            DependencyTimeout.

        retries : int
            Number of times a task will be retried on failure.
        """

        super(LoadBalancedView, self).set_flags(**kwargs)
        for name in ('follow', 'after'):
            if name in kwargs:
                value = kwargs[name]
                if self._validate_dependency(value):
                    setattr(self, name, value)
                else:
                    raise ValueError("Invalid dependency: %r"%value)
        if 'timeout' in kwargs:
            t = kwargs['timeout']
            if not isinstance(t, (int, float, type(None))):
                if (not PY3) and (not isinstance(t, long)):
                    raise TypeError("Invalid type for timeout: %r"%type(t))
            if t is not None:
                if t < 0:
                    raise ValueError("Invalid timeout: %s"%t)
            self.timeout = t

    @sync_results
    @save_ids
    def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
                                        after=None, follow=None, timeout=None,
                                        targets=None, retries=None):
        """calls f(*args, **kwargs) on a remote engine, returning the result.

        This method temporarily sets all of `apply`'s flags for a single call.

        Parameters
        ----------

        f : callable

        args : list [default: empty]

        kwargs : dict [default: empty]

        block : bool [default: self.block]
            whether to block
        track : bool [default: self.track]
            whether to ask zmq to track the message, for safe non-copying sends

        !!!!!! TODO: THE REST HERE  !!!!

        Returns
        -------

        if self.block is False:
            returns AsyncResult
        else:
            returns actual result of f(*args, **kwargs) on the engine(s)
            This will be a list of self.targets is also a list (even length 1), or
            the single result if self.targets is an integer engine id
        """

        # validate whether we can run
        if self._socket.closed():
            msg = "Task farming is disabled"
            if self._task_scheme == 'pure':
                msg += " because the pure ZMQ scheduler cannot handle"
                msg += " disappearing engines."
            raise RuntimeError(msg)

        if self._task_scheme == 'pure':
            # pure zmq scheme doesn't support extra features
            msg = "Pure ZMQ scheduler doesn't support the following flags:"
            "follow, after, retries, targets, timeout"
            if (follow or after or retries or targets or timeout):
                # hard fail on Scheduler flags
                raise RuntimeError(msg)
            if isinstance(f, dependent):
                # soft warn on functional dependencies
                warnings.warn(msg, RuntimeWarning)

        # build args
        args = [] if args is None else args
        kwargs = {} if kwargs is None else kwargs
        block = self.block if block is None else block
        track = self.track if track is None else track
        after = self.after if after is None else after
        retries = self.retries if retries is None else retries
        follow = self.follow if follow is None else follow
        timeout = self.timeout if timeout is None else timeout
        targets = self.targets if targets is None else targets

        if not isinstance(retries, int):
            raise TypeError('retries must be int, not %r'%type(retries))

        if targets is None:
            idents = []
        else:
            idents = self.client._build_targets(targets)[0]
            # ensure *not* bytes
            idents = [ ident.decode() for ident in idents ]

        after = self._render_dependency(after)
        follow = self._render_dependency(follow)
        metadata = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)

        future = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
                                metadata=metadata)

        ar = AsyncResult(self.client, future, fname=getname(f),
            targets=None, owner=True,
        )
        if block:
            try:
                return ar.get()
            except KeyboardInterrupt:
                pass
        return ar

    @sync_results
    @save_ids
    def map(self, f, *sequences, **kwargs):
        """``view.map(f, *sequences, block=self.block, chunksize=1, ordered=True)`` => list|AsyncMapResult

        Parallel version of builtin `map`, load-balanced by this View.

        `block`, and `chunksize` can be specified by keyword only.

        Each `chunksize` elements will be a separate task, and will be
        load-balanced. This lets individual elements be available for iteration
        as soon as they arrive.

        Parameters
        ----------

        f : callable
            function to be mapped
        *sequences: one or more sequences of matching length
            the sequences to be distributed and passed to `f`
        block : bool [default self.block]
            whether to wait for the result or not
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.
        chunksize : int [default 1]
            how many elements should be in each task.
        ordered : bool [default True]
            Whether the results should be gathered as they arrive, or enforce
            the order of submission.
            
            Only applies when iterating through AsyncMapResult as results arrive.
            Has no effect when block=True.

        Returns
        -------

        if block=False
          An :class:`~ipyparallel.client.asyncresult.AsyncMapResult` instance.
          An object like AsyncResult, but which reassembles the sequence of results
          into a single list. AsyncMapResults can be iterated through before all
          results are complete.
        else
            A list, the result of ``map(f,*sequences)``
        """

        # default
        block = kwargs.get('block', self.block)
        chunksize = kwargs.get('chunksize', 1)
        ordered = kwargs.get('ordered', True)

        keyset = set(kwargs.keys())
        extra_keys = keyset.difference_update(set(['block', 'chunksize']))
        if extra_keys:
            raise TypeError("Invalid kwargs: %s"%list(extra_keys))

        assert len(sequences) > 0, "must have some sequences to map onto!"

        pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
        return pf.map(*sequences)

    def register_joblib_backend(self, name='ipyparallel', make_default=False):
        """Register this View as a joblib parallel backend

        To make this the default backend, set make_default=True.

        Use with::

            p = Parallel(backend='ipyparallel')
            ...

        See joblib docs for details

        Requires joblib >= 0.10

        .. versionadded:: 5.1
        """
        from joblib.parallel import register_parallel_backend
        from ._joblib import IPythonParallelBackend
        register_parallel_backend(name,
            lambda : IPythonParallelBackend(view=self),
            make_default=make_default)
Ejemplo n.º 23
0
class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
    name = u'ipython'
    description = usage.cl_usage
    crash_handler_class = IPAppCrashHandler
    examples = _examples

    flags = Dict(flags)
    aliases = Dict(aliases)
    classes = List()

    def _classes_default(self):
        """This has to be in a method, for TerminalIPythonApp to be available."""
        return [
            InteractiveShellApp,  # ShellApp comes before TerminalApp, because
            self.__class__,  # it will also affect subclasses (e.g. QtConsole)
            TerminalInteractiveShell,
            PromptManager,
            HistoryManager,
            ProfileDir,
            PlainTextFormatter,
            IPCompleter,
            ScriptMagics,
            StoreMagics,
        ]

    subcommands = dict(
        qtconsole=('qtconsole.qtconsoleapp.JupyterQtConsoleApp',
                   """DEPRECATD: Launch the Jupyter Qt Console."""),
        notebook=('notebook.notebookapp.NotebookApp',
                  """DEPRECATED: Launch the Jupyter HTML Notebook Server."""),
        profile=("IPython.core.profileapp.ProfileApp",
                 "Create and manage IPython profiles."),
        kernel=("ipykernel.kernelapp.IPKernelApp",
                "Start a kernel without an attached frontend."),
        console=('jupyter_console.app.ZMQTerminalIPythonApp',
                 """DEPRECATED: Launch the Jupyter terminal-based Console."""),
        locate=('IPython.terminal.ipapp.LocateIPythonApp',
                LocateIPythonApp.description),
        history=('IPython.core.historyapp.HistoryApp',
                 "Manage the IPython history database."),
        nbconvert=('nbconvert.nbconvertapp.NbConvertApp',
                   "DEPRECATED: Convert notebooks to/from other formats."),
        trust=
        ('nbformat.sign.TrustNotebookApp',
         "DEPRECATED: Sign notebooks to trust their potentially unsafe contents at load."
         ),
        kernelspec=('jupyter_client.kernelspecapp.KernelSpecApp',
                    "DEPRECATED: Manage Jupyter kernel specifications."),
    )
    subcommands['install-nbextension'] = (
        "notebook.nbextensions.InstallNBExtensionApp",
        "DEPRECATED: Install Jupyter notebook extension files")

    # *do* autocreate requested profile, but don't create the config file.
    auto_create = Bool(True)
    # configurables
    quick = Bool(
        False,
        config=True,
        help=
        """Start IPython quickly by skipping the loading of config files.""")

    def _quick_changed(self, name, old, new):
        if new:
            self.load_config_file = lambda *a, **kw: None

    display_banner = Bool(
        True,
        config=True,
        help="Whether to display a banner upon starting IPython.")

    # if there is code of files to run from the cmd line, don't interact
    # unless the --i flag (App.force_interact) is true.
    force_interact = Bool(
        False,
        config=True,
        help="""If a command or file is given via the command-line,
        e.g. 'ipython foo.py', start an interactive shell after executing the
        file or command.""")

    def _force_interact_changed(self, name, old, new):
        if new:
            self.interact = True

    def _file_to_run_changed(self, name, old, new):
        if new:
            self.something_to_run = True
        if new and not self.force_interact:
            self.interact = False

    _code_to_run_changed = _file_to_run_changed
    _module_to_run_changed = _file_to_run_changed

    # internal, not-configurable
    interact = Bool(True)
    something_to_run = Bool(False)

    def parse_command_line(self, argv=None):
        """override to allow old '-pylab' flag with deprecation warning"""

        argv = sys.argv[1:] if argv is None else argv

        if '-pylab' in argv:
            # deprecated `-pylab` given,
            # warn and transform into current syntax
            argv = argv[:]  # copy, don't clobber
            idx = argv.index('-pylab')
            warn.warn(
                "`-pylab` flag has been deprecated.\n"
                "    Use `--matplotlib <backend>` and import pylab manually.")
            argv[idx] = '--pylab'

        return super(TerminalIPythonApp, self).parse_command_line(argv)

    @catch_config_error
    def initialize(self, argv=None):
        """Do actions after construct, but before starting the app."""
        super(TerminalIPythonApp, self).initialize(argv)
        if self.subapp is not None:
            # don't bother initializing further, starting subapp
            return
        # print self.extra_args
        if self.extra_args and not self.something_to_run:
            self.file_to_run = self.extra_args[0]
        self.init_path()
        # create the shell
        self.init_shell()
        # and draw the banner
        self.init_banner()
        # Now a variety of things that happen after the banner is printed.
        self.init_gui_pylab()
        self.init_extensions()
        self.init_code()

    def init_shell(self):
        """initialize the InteractiveShell instance"""
        # Create an InteractiveShell instance.
        # shell.display_banner should always be False for the terminal
        # based app, because we call shell.show_banner() by hand below
        # so the banner shows *before* all extension loading stuff.
        self.shell = TerminalInteractiveShell.instance(
            parent=self,
            display_banner=False,
            profile_dir=self.profile_dir,
            ipython_dir=self.ipython_dir,
            user_ns=self.user_ns)
        self.shell.configurables.append(self)

    def init_banner(self):
        """optionally display the banner"""
        if self.display_banner and self.interact:
            self.shell.show_banner()
        # Make sure there is a space below the banner.
        if self.log_level <= logging.INFO: print()

    def _pylab_changed(self, name, old, new):
        """Replace --pylab='inline' with --pylab='auto'"""
        if new == 'inline':
            warn.warn("'inline' not available as pylab backend, "
                      "using 'auto' instead.")
            self.pylab = 'auto'

    def start(self):
        if self.subapp is not None:
            return self.subapp.start()
        # perform any prexec steps:
        if self.interact:
            self.log.debug("Starting IPython's mainloop...")
            self.shell.mainloop()
        else:
            self.log.debug("IPython not interactive...")
Ejemplo n.º 24
0
class PluginBlock(Block):
    _view_name = Unicode('PluginBlockView').tag(sync=True)
    _model_name = Unicode('PluginBlockModel').tag(sync=True)

    _parent_block = Instance(BlockType, allow_none=True, default_value=None)

    _available_input_data = List([])
    _available_input_components = List([])
    _input_data_dim = Int(allow_none=True, default_value=None)

    # TODO Validate data/components names and synchronise JavaScript -> Python
    input_data = Unicode(allow_none=True, default_value=None).tag(sync=True)
    input_components = List(Union((Unicode(), Int()))).tag(sync=True)

    def __init__(self, *args, **kwargs):
        super(PluginBlock, self).__init__(*args, **kwargs)
        self.input_data_wid = None
        self.input_components_wid = None

    def _ipython_display_(self, *args, **kwargs):
        display(self.interact())

    def interact(self):
        pass

    def _get_data(self, parent):
        block = parent
        while not isinstance(block, DataBlock):
            block = block._parent_block
        return block.mesh.data

    @observe('_parent_block')
    def _update_input_data(self, change):
        parent = change['new']
        if parent is None:
            return

        data = self._get_data(parent)

        self._available_input_data = [d.name for d in data]
        self.input_data = self._available_input_data[0]

    @observe('input_data')
    def _update_available_components(self, change):
        data = self._get_data(self._parent_block)
        for d in data:
            if d.name == change['new']:
                current_data = d
        self._available_input_components = [c.name for c in current_data.components] + [0]

    @observe('_available_input_components')
    def _update_input_components(self, change):
        if self._input_data_dim is None:
            return

        available_components = change['new']

        if self.input_components_wid is not None:
            for component_wid in self.input_components_wid:
                component_wid.options = available_components

        # Check current components validity
        components_are_valid = True
        if not len(self.input_components):
            components_are_valid = False
        for c in self.input_components:
            if c not in available_components:
                components_are_valid = False
        if components_are_valid:
            return

        new_components = []
        for dim in range(self._input_data_dim):
            if len(available_components) <= dim:
                new_components.append(0)
                continue
            new_components.append(available_components[dim])

        self.input_components = new_components

    def _link_dropdown(self, dropdown, dim):
        def handle_dropdown_change(change):
            copy = self.input_components.copy()
            copy[dim] = change['new']
            self.input_components = copy
        dropdown.observe(handle_dropdown_change, names=['value'])

        def handle_input_change(change):
            dropdown.value = self.input_components[dim]
        self.observe(handle_input_change, names=['input_components'])

        link((dropdown, 'options'), (self, '_available_input_components'))

    def _init_input_data_widgets(self):
        self.input_components_wid = [Label('Input components')]
        for dim in range(self._input_data_dim):
            dropdown = Dropdown(
                options=self._available_input_components,
                value=self.input_components[dim]
            )
            dropdown.layout.width = 'fit-content'
            self._link_dropdown(dropdown, dim)
            self.input_components_wid.append(dropdown)

        self.input_data_wid = Dropdown(
            description='Input data',
            options=self._available_input_data,
            value=self.input_data
        )
        self.input_data_wid.layout.width = 'fit-content'

        link((self.input_data_wid, 'value'), (self, 'input_data'))

    def _interact(self):
        if self._input_data_dim is not None:
            if self.input_data_wid is None:
                self._init_input_data_widgets()

            return (VBox((self.input_data_wid, HBox(self.input_components_wid))), )

        return ()

    def _get_component_min_max(self, data_name, component_name):
        data = self._get_data(self._parent_block)
        for d in data:
            if d.name == data_name:
                for c in d.components:
                    if c.name == component_name:
                        return (c.min, c.max)
        raise RuntimeError('Unknown component {}.{}'.format(
            data_name, component_name))
Ejemplo n.º 25
0
class Circle(Path):
    _view_name = Unicode('LeafletCircleView').tag(sync=True)
    _model_name = Unicode('LeafletCircleModel').tag(sync=True)

    location = List(def_loc).tag(sync=True)
    radius = Int(1000, help="radius of circle in meters").tag(sync=True)
Ejemplo n.º 26
0
class Block(Widget, BlockType):
    _view_name = Unicode('BlockView').tag(sync=True)
    _model_name = Unicode('BlockModel').tag(sync=True)
    _view_module = Unicode('odysis').tag(sync=True)
    _model_module = Unicode('odysis').tag(sync=True)
    _view_module_version = Unicode(odysis_version).tag(sync=True)
    _model_module_version = Unicode(odysis_version).tag(sync=True)

    _blocks = List(Instance(BlockType)).tag(sync=True, **widget_serialization)

    visible = Bool(True).tag(sync=True)

    def apply(self, block):
        block._validate_parent(self)

        if block._parent_block is not None:
            raise RuntimeError('Cannot apply the same effect at different places')

        block._parent_block = self
        self._blocks = list([b for b in self._blocks] + [block])

    def remove(self, block):
        block._parent_block = None
        self._blocks = list([b for b in self._blocks if b.model_id != block.model_id])

    def color_mapping(self, *args, **kwargs):
        effect = ColorMapping(*args, **kwargs)
        self.apply(effect)
        return effect

    def grid(self, *args, **kwargs):
        effect = Grid(*args, **kwargs)
        self.apply(effect)
        return effect

    def warp(self, *args, **kwargs):
        effect = Warp(*args, **kwargs)
        self.apply(effect)
        return effect

    def vector_field(self, *args, **kwargs):
        effect = VectorField(*args, **kwargs)
        self.apply(effect)
        return effect

    def point_cloud(self, *args, **kwargs):
        effect = PointCloud(*args, **kwargs)
        self.apply(effect)
        return effect

    def clip(self, *args, **kwargs):
        effect = Clip(*args, **kwargs)
        self.apply(effect)
        return effect

    def slice(self, *args, **kwargs):
        effect = Slice(*args, **kwargs)
        self.apply(effect)
        return effect

    def threshold(self, *args, **kwargs):
        effect = Threshold(*args, **kwargs)
        self.apply(effect)
        return effect

    def iso_surface(self, *args, **kwargs):
        effect = IsoSurface(*args, **kwargs)
        self.apply(effect)
        return effect

    def __init__(self, *args, **kwargs):
        super(Block, self).__init__(*args, **kwargs)
        self.colormap_wid = None
        self.colormapslider_wid = None

    def _validate_parent(self, parent):
        pass
Ejemplo n.º 27
0
class MultiPolygon(FeatureGroup):
    _view_name = Unicode('LeafletMultiPolygonView').tag(sync=True)
    _model_name = Unicode('LeafletMultiPolygonModel').tag(sync=True)
    locations = List().tag(sync=True)
Ejemplo n.º 28
0
class LabConfig(HasTraits):
    """The lab application configuration object.
    """
    app_name = Unicode('', help='The name of the application.')

    app_version = Unicode('', help='The version of the application.')

    app_namespace = Unicode('', help='The namespace of the application.')

    app_url = Unicode('/lab', help='The url path for the application.')

    app_settings_dir = Unicode('', help='The application settings directory.')

    extra_labextensions_path = List(
        Unicode(),
        help="""Extra paths to look for dynamic JupyterLab extensions""")

    labextensions_path = List(
        Unicode(),
        help='The standard paths to look in for dynamic JupyterLab extensions')

    templates_dir = Unicode('', help='The application templates directory.')

    static_dir = Unicode('',
                         help=('The optional location of local static files. '
                               'If given, a static file handler will be '
                               'added.'))

    static_url = Unicode(help=('The url path for static application '
                               'assets. This can be a CDN if desired.'))

    labextensions_url = Unicode(
        '', help='The url for dynamic JupyterLab extensions')

    settings_url = Unicode(help='The url path of the settings handler.')

    user_settings_dir = Unicode('',
                                help=('The optional location of the user '
                                      'settings directory.'))

    schemas_dir = Unicode('',
                          help=('The optional location of the settings '
                                'schemas directory. If given, a handler will '
                                'be added for settings.'))

    workspaces_api_url = Unicode(help='The url path of the workspaces API.')

    workspaces_dir = Unicode('',
                             help=('The optional location of the saved '
                                   'workspaces directory. If given, a handler '
                                   'will be added for workspaces.'))

    listings_url = Unicode(help='The listings url.')

    themes_url = Unicode(help='The theme url.')

    themes_dir = Unicode('',
                         help=('The optional location of the themes '
                               'directory. If given, a handler will be added '
                               'for themes.'))

    translations_api_url = Unicode(
        help='The url path of the translations handler.')

    tree_url = Unicode(help='The url path of the tree handler.')

    cache_files = Bool(True,
                       help=('Whether to cache files on the server. '
                             'This should be `True` except in dev mode.'))

    @default('template_dir')
    def _default_template_dir(self):
        return DEFAULT_TEMPLATE_PATH

    @default('static_url')
    def _default_static_url(self):
        return ujoin('static/', self.app_namespace)

    @default('labextensions_url')
    def _default_labextensions_url(self):
        return ujoin(self.app_url, "extensions/")

    @default('labextensions_path')
    def _default_labextensions_path(self):
        return jupyter_path('labextensions')

    @default('workspaces_url')
    def _default_workspaces_url(self):
        return ujoin(self.app_url, 'workspaces/')

    @default('workspaces_api_url')
    def _default_workspaces_api_url(self):
        return ujoin(self.app_url, 'api', 'workspaces/')

    @default('settings_url')
    def _default_settings_url(self):
        return ujoin(self.app_url, 'api', 'settings/')

    @default('listings_url')
    def _default_listings_url(self):
        return ujoin(self.app_url, 'api', 'listings/')

    @default('themes_url')
    def _default_themes_url(self):
        return ujoin(self.app_url, 'api', 'themes/')

    @default('tree_url')
    def _default_tree_url(self):
        return ujoin(self.app_url, 'tree/')

    @default('translations_api_url')
    def _default_translations_api_url(self):
        return ujoin(self.app_url, 'api', 'translations/')

    @default('tree_url')
    def _default_tree_url(self):
        return ujoin(self.app_url, 'tree/')
Ejemplo n.º 29
0
class View(HasTraits):
    """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.

    Don't use this class, use subclasses.

    Methods
    -------

    spin
        flushes incoming results and registration state changes
        control methods spin, and requesting `ids` also ensures up to date

    wait
        wait on one or more msg_ids

    execution methods
        apply
        legacy: execute, run

    data movement
        push, pull, scatter, gather

    query methods
        get_result, queue_status, purge_results, result_status

    control methods
        abort, shutdown

    """
    # flags
    block = Bool(False)
    track = Bool(False)
    targets = Any()

    history = List()
    outstanding = Set()
    results = Dict()
    client = Instance('ipyparallel.Client', allow_none=True)

    _socket = Instance('zmq.Socket', allow_none=True)
    _flag_names = List(['targets', 'block', 'track'])
    _in_sync_results = Bool(False)
    _targets = Any()
    _idents = Any()

    def __init__(self, client=None, socket=None, **flags):
        super(View, self).__init__(client=client, _socket=socket)
        self.results = client.results
        self.block = client.block
        self.executor = ViewExecutor(self)

        self.set_flags(**flags)

        assert not self.__class__ is View, "Don't use base View objects, use subclasses"

    def __repr__(self):
        strtargets = str(self.targets)
        if len(strtargets) > 16:
            strtargets = strtargets[:12]+'...]'
        return "<%s %s>"%(self.__class__.__name__, strtargets)
    
    def __len__(self):
        if isinstance(self.targets, list):
            return len(self.targets)
        elif isinstance(self.targets, int):
            return 1
        else:
            return len(self.client)
    
    def set_flags(self, **kwargs):
        """set my attribute flags by keyword.

        Views determine behavior with a few attributes (`block`, `track`, etc.).
        These attributes can be set all at once by name with this method.

        Parameters
        ----------

        block : bool
            whether to wait for results
        track : bool
            whether to create a MessageTracker to allow the user to
            safely edit after arrays and buffers during non-copying
            sends.
        """
        for name, value in iteritems(kwargs):
            if name not in self._flag_names:
                raise KeyError("Invalid name: %r"%name)
            else:
                setattr(self, name, value)

    @contextmanager
    def temp_flags(self, **kwargs):
        """temporarily set flags, for use in `with` statements.

        See set_flags for permanent setting of flags

        Examples
        --------

        >>> view.track=False
        ...
        >>> with view.temp_flags(track=True):
        ...    ar = view.apply(dostuff, my_big_array)
        ...    ar.tracker.wait() # wait for send to finish
        >>> view.track
        False

        """
        # preflight: save flags, and set temporaries
        saved_flags = {}
        for f in self._flag_names:
            saved_flags[f] = getattr(self, f)
        self.set_flags(**kwargs)
        # yield to the with-statement block
        try:
            yield
        finally:
            # postflight: restore saved flags
            self.set_flags(**saved_flags)


    #----------------------------------------------------------------
    # apply
    #----------------------------------------------------------------

    def _sync_results(self):
        """to be called by @sync_results decorator
        
        after submitting any tasks.
        """
        delta = self.outstanding.difference(self.client.outstanding)
        completed = self.outstanding.intersection(delta)
        self.outstanding = self.outstanding.difference(completed)
    
    @sync_results
    @save_ids
    def _really_apply(self, f, args, kwargs, block=None, **options):
        """wrapper for client.send_apply_request"""
        raise NotImplementedError("Implement in subclasses")

    def apply(self, f, *args, **kwargs):
        """calls ``f(*args, **kwargs)`` on remote engines, returning the result.

        This method sets all apply flags via this View's attributes.

        Returns :class:`~ipyparallel.client.asyncresult.AsyncResult`
        instance if ``self.block`` is False, otherwise the return value of
        ``f(*args, **kwargs)``.
        """
        return self._really_apply(f, args, kwargs)

    def apply_async(self, f, *args, **kwargs):
        """calls ``f(*args, **kwargs)`` on remote engines in a nonblocking manner.

        Returns :class:`~ipyparallel.client.asyncresult.AsyncResult` instance.
        """
        return self._really_apply(f, args, kwargs, block=False)

    def apply_sync(self, f, *args, **kwargs):
        """calls ``f(*args, **kwargs)`` on remote engines in a blocking manner,
         returning the result.
        """
        return self._really_apply(f, args, kwargs, block=True)

    #----------------------------------------------------------------
    # wrappers for client and control methods
    #----------------------------------------------------------------
    @sync_results
    def spin(self):
        """spin the client, and sync"""
        self.client.spin()

    @sync_results
    def wait(self, jobs=None, timeout=-1):
        """waits on one or more `jobs`, for up to `timeout` seconds.

        Parameters
        ----------

        jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
                ints are indices to self.history
                strs are msg_ids
                default: wait on all outstanding messages
        timeout : float
                a time in seconds, after which to give up.
                default is -1, which means no timeout

        Returns
        -------

        True : when all msg_ids are done
        False : timeout reached, some msg_ids still outstanding
        """
        if jobs is None:
            jobs = self.history
        return self.client.wait(jobs, timeout)

    def abort(self, jobs=None, targets=None, block=None):
        """Abort jobs on my engines.

        Parameters
        ----------

        jobs : None, str, list of strs, optional
            if None: abort all jobs.
            else: abort specific msg_id(s).
        """
        block = block if block is not None else self.block
        targets = targets if targets is not None else self.targets
        jobs = jobs if jobs is not None else list(self.outstanding)
        
        return self.client.abort(jobs=jobs, targets=targets, block=block)

    def queue_status(self, targets=None, verbose=False):
        """Fetch the Queue status of my engines"""
        targets = targets if targets is not None else self.targets
        return self.client.queue_status(targets=targets, verbose=verbose)

    def purge_results(self, jobs=[], targets=[]):
        """Instruct the controller to forget specific results."""
        if targets is None or targets == 'all':
            targets = self.targets
        return self.client.purge_results(jobs=jobs, targets=targets)

    def shutdown(self, targets=None, restart=False, hub=False, block=None):
        """Terminates one or more engine processes, optionally including the hub.
        """
        block = self.block if block is None else block
        if targets is None or targets == 'all':
            targets = self.targets
        return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)

    def get_result(self, indices_or_msg_ids=None, block=None, owner=False):
        """return one or more results, specified by history index or msg_id.

        See :meth:`ipyparallel.client.client.Client.get_result` for details.
        """

        if indices_or_msg_ids is None:
            indices_or_msg_ids = -1
        if isinstance(indices_or_msg_ids, int):
            indices_or_msg_ids = self.history[indices_or_msg_ids]
        elif isinstance(indices_or_msg_ids, (list,tuple,set)):
            indices_or_msg_ids = list(indices_or_msg_ids)
            for i,index in enumerate(indices_or_msg_ids):
                if isinstance(index, int):
                    indices_or_msg_ids[i] = self.history[index]
        return self.client.get_result(indices_or_msg_ids, block=block, owner=owner)

    #-------------------------------------------------------------------
    # Map
    #-------------------------------------------------------------------

    @sync_results
    def map(self, f, *sequences, **kwargs):
        """override in subclasses"""
        raise NotImplementedError

    def map_async(self, f, *sequences, **kwargs):
        """Parallel version of builtin :func:`python:map`, using this view's engines.

        This is equivalent to ``map(...block=False)``.

        See `self.map` for details.
        """
        if 'block' in kwargs:
            raise TypeError("map_async doesn't take a `block` keyword argument.")
        kwargs['block'] = False
        return self.map(f,*sequences,**kwargs)

    def map_sync(self, f, *sequences, **kwargs):
        """Parallel version of builtin :func:`python:map`, using this view's engines.

        This is equivalent to ``map(...block=True)``.

        See `self.map` for details.
        """
        if 'block' in kwargs:
            raise TypeError("map_sync doesn't take a `block` keyword argument.")
        kwargs['block'] = True
        return self.map(f,*sequences,**kwargs)

    def imap(self, f, *sequences, **kwargs):
        """Parallel version of :func:`itertools.imap`.

        See `self.map` for details.

        """

        return iter(self.map_async(f,*sequences, **kwargs))

    #-------------------------------------------------------------------
    # Decorators
    #-------------------------------------------------------------------

    def remote(self, block=None, **flags):
        """Decorator for making a RemoteFunction"""
        block = self.block if block is None else block
        return remote(self, block=block, **flags)

    def parallel(self, dist='b', block=None, **flags):
        """Decorator for making a ParallelFunction"""
        block = self.block if block is None else block
        return parallel(self, dist=dist, block=block, **flags)
Ejemplo n.º 30
0
class SimpleWidget(Widget):
    a = Bool().tag(sync=True)
    b = Tuple(Bool(), Bool(), Bool(),
              default_value=(False, False, False)).tag(sync=True)
    c = List(Bool()).tag(sync=True)