class AnimationWidget(widgets.DOMWidget): """ A widget that periodic increment a value :param value: A float between 0 and 1 :param run: boolean with the state of the timer. True, the timer is enable Produces the following signal. A sampling rate, the value is interpolated with the equation val = 1/Period * t 1- ^ ____ | / |/ 0- |-----> | period """ _view_name = traitlets.Unicode('AnimationView').tag(sync=True) _model_name = traitlets.Unicode('AnimationModel').tag(sync=True) _view_module = traitlets.Unicode('animation-widget').tag(sync=True) _model_module = traitlets.Unicode('animation-widget').tag(sync=True) # Signal value value = traitlets.CFloat(0.0).tag(sync=True) # Boolean timer is active run = traitlets.CBool(False).tag(sync=True) # Signal period (in ms) period = traitlets.CFloat(5000).tag(sync=True) # Number of samples in period nbsamples = traitlets.CInt(100).tag(sync=True) # Loop loop = traitlets.CBool(False).tag(sync=True)
class Scatter(widgets.Widget): _view_name = Unicode('ScatterView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('ScatterModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) vx = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) vy = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) vz = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) sequence_index = Integer(default_value=0).tag(sync=True) size = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization), traitlets.Float().tag(sync=True)], default_value=5).tag(sync=True) size_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization), traitlets.Float().tag(sync=True)], default_value=7).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization), Unicode().tag(sync=True)], default_value="green").tag(sync=True) geo = traitlets.Unicode('diamond').tag(sync=True) connected = traitlets.CBool(default_value=False).tag(sync=True) visible = traitlets.CBool(default_value=True).tag(sync=True) texture = traitlets.Union([ traitlets.Instance(ipywebrtc.MediaStream), Unicode(), traitlets.List(Unicode, [], allow_none=True), Image(default_value=None, allow_none=True), traitlets.List(Image(default_value=None, allow_none=True)) ]).tag(sync=True, **texture_serialization) material = traitlets.Instance(pythreejs.ShaderMaterial, help='A :any:`pythreejs.ShaderMaterial` that is used for the mesh')\ .tag(sync=True, **ipywidgets.widget_serialization) @traitlets.default('material') def _default_material(self): return pythreejs.ShaderMaterial() line_material = traitlets.Instance(pythreejs.ShaderMaterial, help='A :any:`pythreejs.ShaderMaterial` that is used for the lines/wireframe')\ .tag(sync=True, **ipywidgets.widget_serialization) @traitlets.default('line_material') def _default_line_material(self): return pythreejs.ShaderMaterial()
class Scatter(widgets.DOMWidget): _view_name = Unicode('ScatterView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('ScatterModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) vx = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) vy = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) vz = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) sequence_index = Integer(default_value=0).tag(sync=True) size = traitlets.Union([ Array(default_value=None, allow_none=True).tag( sync=True, **array_sequence_serialization), traitlets.Float().tag(sync=True) ], default_value=5).tag(sync=True) size_selected = traitlets.Union([ Array(default_value=None, allow_none=True).tag( sync=True, **array_sequence_serialization), traitlets.Float().tag(sync=True) ], default_value=7).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) color_selected = traitlets.Union([ Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization), Unicode().tag(sync=True) ], default_value="green").tag(sync=True) geo = traitlets.Unicode('diamond').tag(sync=True) connected = traitlets.CBool(default_value=False).tag(sync=True) visible = traitlets.CBool(default_value=True).tag(sync=True) visible_lines = traitlets.CBool(default_value=False).tag(sync=True) visible_markers = traitlets.CBool(default_value=True).tag(sync=True)
class Mesh(widgets.DOMWidget): _view_name = Unicode('MeshView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('MeshModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) u = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) v = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) triangles = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) lines = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) texture = Image(default_value=None, allow_none=True).tag(sync=True, **image_serialization) # selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) sequence_index = Integer(default_value=0).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) # color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization), # Unicode().tag(sync=True)], # default_value="green").tag(sync=True) # geo = traitlets.Unicode('diamond').tag(sync=True) wire = traitlets.CBool(default_value=False).tag(sync=True)
class Mesh(widgets.DOMWidget): _view_name = Unicode('MeshView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('MeshModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) u = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) v = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) triangles = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) lines = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) texture = traitlets.Union([ traitlets.Instance(ipywebrtc.MediaStream), Unicode(), traitlets.List(Unicode, [], allow_none=True), Image(default_value=None, allow_none=True), traitlets.List(Image(default_value=None, allow_none=True)) ]).tag(sync=True, **texture_serialization) # selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) sequence_index = Integer(default_value=0).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) # color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization), # Unicode().tag(sync=True)], # default_value="green").tag(sync=True) # geo = traitlets.Unicode('diamond').tag(sync=True) visible = traitlets.CBool(default_value=True).tag(sync=True) visible_lines = traitlets.CBool(default_value=True).tag(sync=True) visible_faces = traitlets.CBool(default_value=True).tag(sync=True) side = traitlets.CaselessStrEnum(['front', 'back', 'both'], 'both').tag(sync=True)
class Mesh(widgets.Widget): _view_name = Unicode('MeshView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('MeshModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) u = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) v = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) triangles = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) lines = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) texture = traitlets.Union([ traitlets.Instance(ipywebrtc.MediaStream), Unicode(), traitlets.List(Unicode, [], allow_none=True), Image(default_value=None, allow_none=True), traitlets.List(Image(default_value=None, allow_none=True)), ]).tag(sync=True, **texture_serialization) sequence_index = Integer(default_value=0).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) visible = traitlets.CBool(default_value=True).tag(sync=True) material = traitlets.Instance( pythreejs.ShaderMaterial, help='A :any:`pythreejs.ShaderMaterial` that is used for the mesh' ).tag(sync=True, **widgets.widget_serialization) @traitlets.default('material') def _default_material(self): return pythreejs.ShaderMaterial(side=pythreejs.enums.Side.DoubleSide) line_material = traitlets.Instance( pythreejs.ShaderMaterial, help= 'A :any:`pythreejs.ShaderMaterial` that is used for the lines/wireframe' ).tag(sync=True, **widgets.widget_serialization) @traitlets.default('line_material') def _default_line_material(self): return pythreejs.ShaderMaterial()
class Mesh(widgets.DOMWidget): _view_name = Unicode('MeshView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('MeshModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) x = Array(default_value=None).tag(sync=True, **array_sequence_serialization) y = Array(default_value=None).tag(sync=True, **array_sequence_serialization) z = Array(default_value=None).tag(sync=True, **array_sequence_serialization) u = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) v = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) triangles = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) lines = Array(default_value=None, allow_none=True).tag(sync=True, **array_serialization) texture = traitlets.Union([ traitlets.Instance(ipywebrtc.MediaStream), Unicode(), traitlets.List(Unicode, [], allow_none=True), Image(default_value=None, allow_none=True), traitlets.List(Image(default_value=None, allow_none=True)) ]).tag(sync=True, **texture_serialization) # selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization) sequence_index = Integer(default_value=0).tag(sync=True) color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization) # color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization), # Unicode().tag(sync=True)], # default_value="green").tag(sync=True) # geo = traitlets.Unicode('diamond').tag(sync=True) visible = traitlets.CBool(default_value=True).tag(sync=True) material = traitlets.Instance(pythreejs.ShaderMaterial).tag( sync=True, **ipywidgets.widget_serialization) @traitlets.default('material') def _default_material(self): return pythreejs.ShaderMaterial(side=pythreejs.Side.DoubleSide) line_material = traitlets.Instance(pythreejs.ShaderMaterial).tag( sync=True, **ipywidgets.widget_serialization) @traitlets.default('line_material') def _default_line_material(self): return pythreejs.ShaderMaterial()
class Branch(t.HasTraits): "Branch model" name = t.CUnicode(default_value='Branch1', help='Name of Branch (str)') from_bus = t.CUnicode(default_value='Bus1', help='Name of From Bus') to_bus = t.CUnicode(default_value='Bus2', help='Name of To Bus') resistance = t.CFloat(default_value=0, help='Branch resistance (p.u.)') reactance = t.CFloat(default_value=0, help='Branch reactance (p.u.)') susceptance = t.CFloat(default_value=0, help='Branch susceptance (p.u.)') rating = t.CFloat(default_value=M, help='Branch Rating') status = t.CBool(default_value=1, help='Branch status') angle_minimum = t.CFloat(default_value=0.0, help='Branch angle minimum') angle_maximum = t.CFloat(default_value=0.0, help='Branch angle maximum') tap = t.CFloat(default_value=None, allow_none=True) shift = t.CFloat(default_value=None, allow_none=True) from_real_power_flow = t.CFloat(default_value=None, allow_none=True, help='From active power flow (MW)') from_imag_power_flow = t.CFloat(default_value=None, allow_none=True, help='From reactive power flow (MVAR)') to_real_power_flow = t.CFloat(default_value=None, allow_none=True, help='To active power flow (MW)') to_imag_power_flow = t.CFloat(default_value=None, allow_none=True, help='To reactive power flow (MVAR)')
class StandardScaler(Transformer): '''Standardize features by removing thir mean and scaling them to unit variance. Example: >>> import vaex >>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10]) >>> df # x y 0 2 -2 1 5 3 2 7 0 3 2 0 4 15 10 >>> scaler = vaex.ml.StandardScaler(features=['x', 'y']) >>> scaler.fit_transform(df) # x y standard_scaled_x standard_scaled_y 0 2 -2 -0.876523 -0.996616 1 5 3 -0.250435 0.189832 2 7 0 0.166957 -0.522037 3 2 0 -0.876523 -0.522037 4 15 10 1.83652 1.85086 ''' # title = Unicode(default_value='Standard Scaler', read_only=True).tag(ui='HTML') prefix = traitlets.Unicode(default_value="standard_scaled_", help=help_prefix).tag(ui='Text') with_mean = traitlets.CBool( default_value=True, help='If True, remove the mean from each feature.').tag(ui='Checkbox') with_std = traitlets.CBool( default_value=True, help='If True, scale each feature to unit variance.').tag( ui='Checkbox') mean_ = traitlets.List(traitlets.CFloat(), help='The mean of each feature').tag(output=True) std_ = traitlets.List( traitlets.CFloat(), help='The standard deviation of each feature.').tag(output=True) def fit(self, df): ''' Fit StandardScaler to the DataFrame. :param df: A vaex DataFrame. ''' mean = df.mean(self.features, delay=True) std = df.std(self.features, delay=True) @vaex.delayed def assign(mean, std): self.mean_ = mean.tolist() self.std_ = std.tolist() assign(mean, std) df.execute() def transform(self, df): ''' Transform a DataFrame with a fitted StandardScaler. :param df: A vaex DataFrame. :returns copy: a shallow copy of the DataFrame that includes the scaled features. :rtype: DataFrame ''' copy = df.copy() for i, feature in enumerate(self.features): name = self.prefix + feature expression = copy[feature] if self.with_mean: expression -= self.mean_[i] if self.with_std: expression /= self.std_[i] copy[name] = expression return copy
class Plot2dSliced(PlotBase): z = traitlets.Unicode(allow_none=False).tag(sync=True) z_slice = traitlets.CInt(default_value=0).tag( sync=True) #.tag(sync=True) # TODO: do linking at python side z_shape = traitlets.CInt(default_value=10).tag(sync=True) z_relative = traitlets.CBool(False).tag(sync=True) z_min = traitlets.CFloat(default_value=None, allow_none=True).tag(sync=True) #.tag(sync=True) z_max = traitlets.CFloat(default_value=None, allow_none=True).tag(sync=True) #.tag(sync=True) def __init__(self, **kwargs): self.z_min_extreme, self.z_max_extreme = kwargs["dataset"].minmax( kwargs["z"]) super(Plot2dSliced, self).__init__(**kwargs) self.create_tools() def get_limits(self, limits): limits = self.dataset.limits(self.get_binby(), limits) limits = list([list(k) for k in limits]) if self.z_min is None: self.z_min = limits[2][0] if self.z_max is None: self.z_max = limits[2][1] limits[2][0] = self.z_min limits[2][1] = self.z_max return limits def select_rectangle(self, x1, y1, x2, y2, mode="replace"): dz = self.z_max - self.z_min z1 = self.z_min + dz * self.z_slice / self.z_shape z2 = self.z_min + dz * (self.z_slice + 1) / self.z_shape spaces = [self.x, self.y, self.z] limits = [[x1, x2], [y1, y2], [z1, z2]] self.dataset.select_box(spaces, limits=limits, mode=mode) def select_lasso(self, x, y, mode="replace"): raise NotImplementedError("todo") def get_grid(self): zslice = self.grid[..., self.z_slice] if self.z_relative: with np.errstate(divide='ignore', invalid='ignore'): zslice = zslice / self.grid.sum(axis=-1) return zslice #return self.grid[...,self.z_slice] def get_vgrids(self): def zsliced(grid): return grid[..., self.z_slice] if grid is not None else None return [ zsliced(grid) for grid in super(Plot2dSliced, self).get_vgrids() ] def create_tools(self): #super(Plot2dSliced, self).create_tools() self.z_slice_slider = widgets.IntSlider(value=self.z_slice, min=0, max=self.z_shape - 1) #self.add_control_widget(self.z_slice_slider) self.z_slice_slider.observe(self._z_slice_changed, "value") self.observe(self._z_slice_changed, "z_slice") dz = self.z_max_extreme - self.z_min_extreme self.z_range_slider = widgets.FloatRangeSlider( min=min(self.z_min, self.z_min_extreme), value=[self.z_min, self.z_max], max=max(self.z_max, self.z_max_extreme), step=dz / 1000) self.z_range_slider.observe(self._z_range_changed_, names=["value"]) #self.observe(self.z_range_slider, "z_min") self.z_control = widgets.VBox( [self.z_slice_slider, self.z_range_slider]) self.add_control_widget(self.z_control) def _z_range_changed_(self, changes, **kwargs): #print("changes1", changes, repr(changes), kwargs) self.limits[2][0], self.limits[2][1] =\ self.z_min, self.z_max = self.z_range_slider.value = changes["new"] self.update_grid() def _z_slice_changed(self, changes): self.z_slice = self.z_slice_slider.value = changes["new"] self._update_image() def get_shape(self): return vaex.dataset._expand_shape(self.shape, 2) + (self.z_shape, ) def get_vshape(self): return vaex.dataset._expand_shape(self.vshape, 2) + (self.z_shape, ) def get_binby(self): return [self.x, self.y, self.z]
class Volume(widgets.Widget): """Widget class representing a volume (rendering) using three.js.""" _view_name = Unicode('VolumeView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('VolumeModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) data = Array(default_value=None, allow_none=True).tag(sync=True, **array_cube_tile_serialization) data_original = Array(default_value=None, allow_none=True) data_max_shape = traitlets.CInt(None, allow_none=True) # TODO: allow this to be a list data_min = traitlets.CFloat(0).tag(sync=True) data_max = traitlets.CFloat(1).tag(sync=True) show_min = traitlets.CFloat(0).tag(sync=True) show_max = traitlets.CFloat(1).tag(sync=True) clamp_min = traitlets.CBool(False).tag(sync=True) clamp_max = traitlets.CBool(False).tag(sync=True) opacity_scale = traitlets.CFloat(1.0).tag(sync=True) brightness = traitlets.CFloat(1.0).tag(sync=True) tf = traitlets.Instance(TransferFunction, allow_none=True).tag(sync=True, **widgets.widget_serialization) ray_steps = traitlets.CInt( None, allow_none=True, help='defines the length of the ray (1/ray_steps) for each step, in normalized coordintes.', ).tag(sync=True) rendering_method = traitlets.Enum(values=['NORMAL', 'MAX_INTENSITY'], default_value='NORMAL').tag(sync=True) lighting = traitlets.Bool(True).tag(sync=True) extent = traitlets.Any().tag(sync=True) extent_original = traitlets.Any() def __init__(self, **kwargs): super(Volume, self).__init__(**kwargs) self._update_data() self.observe(self.update_data, ['data_original', 'data_max_shape']) def _listen_to(self, fig): fig.observe(self.update_data, ['xlim', 'ylim', 'zlim']) @debounced(method=True) def update_data(self, change=None): self._update_data() def _update_data(self): if self.data_original is None: return if all([k <= self.data_max_shape for k in self.data_original.shape]): self.data = self.data_original self.extent = self.extent_original return current_figure = ipv.gcf() xlim = current_figure.xlim ylim = current_figure.ylim zlim = current_figure.zlim shape = self.data_original.shape ex = self.extent_original viewx, xt = grid_slice(ex[0][0], ex[0][1], shape[2], *xlim) viewy, yt = grid_slice(ex[1][0], ex[1][1], shape[1], *ylim) viewz, zt = grid_slice(ex[2][0], ex[2][1], shape[0], *zlim) view = [slice(*viewz), slice(*viewy), slice(*viewx)] data_view = self.data_original[view] extent = [xt, yt, zt] data_view, extent = reduce_size(data_view, self.data_max_shape, extent) self.data = np.array(data_view) self.extent = extent
class Watchdog(widgets.Widget): path = traitlets.CUnicode(".", sync=True) description = traitlets.CUnicode(sync=True) value = traitlets.Any(sync=True) recursive = traitlets.CBool(False, sync=True) watching = traitlets.CBool(False, sync=True) poll_interval = traitlets.CInt(1, sync=True) last_event = traitlets.Dict(sync=True) file = traitlets.CUnicode(sync=True) def __init__(self, *args, **kwargs): super(Watchdog, self).__init__(*args, **kwargs) self._wd_handlers = widgets.CallbackDispatcher() self._wd_observer = None self.on_msg(self._handle_wd_msg) def __del__(self): self.stop() super(Watchdog, self).__del__() def _handle_wd_msg(self, _, content, buffers): """ Be very careful here: inspecting the messages in the browser may be the only way to find problems """ self.last_event = content event = content.get('event', None) if event is None: return if (not self.file) or fnmatch(content["src_path"], self.file): self._wd_handlers(self, content) def on_any(self, callback, remove=False): self._wd_handlers.register_callback(callback, remove=remove) def ls(self): return glob.glob(os.path.join(self.path, self.file or "*.*")) def touch(self, idx=0): os.utime(self.ls()[idx], None) def start(self): self.stop() ip = get_ipython() self._wd_observer = subprocess.Popen(map(str, [ sys.executable, "-m", "ipywatchdogwidget.observe", "--model-id", self.model_id, "--path", os.path.abspath(self.path), "--poll-interval", self.poll_interval, "--port", ip.kernel._recorded_ports["shell"], "--recursive", self.recursive, "--session-key", ip.kernel.session.key.decode() ]), stderr=subprocess.STDOUT) self.watching = True def stop(self): if self._wd_observer: parent = psutil.Process(self._wd_observer.pid) [child.kill() for child in parent.children(recursive=True)] parent.kill() self._wd_observer = None self.watching = False
class KMeans(vaex.ml.state.HasState): '''The KMeans clustering algorithm. Example: >>> import vaex.ml >>> import vaex.ml.cluster >>> df = vaex.ml.datasets.load_iris() >>> features = ['sepal_width', 'petal_length', 'sepal_length', 'petal_width'] >>> cls = vaex.ml.cluster.KMeans(n_clusters=3, features=features, init='random', max_iter=10) >>> cls.fit(df) >>> df = cls.transform(df) >>> df.head(5) # sepal_width petal_length sepal_length petal_width class_ prediction_kmeans 0 3 4.2 5.9 1.5 1 2 1 3 4.6 6.1 1.4 1 2 2 2.9 4.6 6.6 1.3 1 2 3 3.3 5.7 6.7 2.1 2 0 4 4.2 1.4 5.5 0.2 0 1 ''' features = traitlets.List(traitlets.Unicode(), help='List of features to cluster.') n_clusters = traitlets.CInt(default_value=2, help='Number of clusters to form.') init = traitlets.Union([Matrix(), traitlets.Unicode()], default_value='random', help='Method for initializing the centroids.') n_init = traitlets.CInt(default_value=1, help='Number of centroid initializations. \ The KMeans algorithm will be run for each initialization, \ and the final results will be the best output of the n_init \ consecutive runs in terms of inertia.' ) max_iter = traitlets.CInt( default_value=300, help= 'Maximum number of iterations of the KMeans algorithm for a single run.' ) random_state = traitlets.CInt( default_value=None, allow_none=True, help='Random number generation for centroid initialization. \ If an int is specified, the randomness becomes deterministic.' ) verbose = traitlets.CBool(default_value=False, help='If True, enable verbosity mode.') cluster_centers = traitlets.List(traitlets.List(traitlets.CFloat()), help='Coordinates of cluster centers.') inertia = traitlets.CFloat( default_value=None, allow_none=True, help= 'Sum of squared distances of samples to their closest cluster center.') prediction_label = traitlets.Unicode( default_value='prediction_kmeans', help= 'The name of the virtual column that houses the cluster labels for each point.' ) def __call__(self, *blocks): return self._calculate_classes(*blocks) def _calculate_distances_squared(self, *blocks): N = len(blocks[0]) # they are all the same length centroids = np.array(self.cluster_centers) k = centroids.shape[0] dimensions = centroids.shape[1] distances_sq = np.zeros((k, N)) if True: distances_square(distances_sq, centroids, *blocks) else: for d in range(dimensions): for i in range(k): distances_sq[i] += (blocks[d] - centroids[i][d])**2 return distances_sq def _calculate_classes(self, *blocks): distances_sq = self._calculate_distances_squared(*blocks) classes = np.argmin(distances_sq, axis=0) return classes def generate_cluster_centers_random(self, dataframe, rng): indices = rng.randint(0, len(dataframe), self.n_clusters) return [[ dataframe.evaluate(feature, i1=i, i2=i + 1)[0] for feature in self.features ] for i in indices] def transform(self, dataframe): ''' Label a DataFrame with a fitted KMeans model. :param dataframe: A vaex DataFrame. :returns copy: A shallow copy of the DataFrame that includes the cluster labels. :rtype: DataFrame ''' copy = dataframe.copy() lazy_function = copy.add_function('kmean_predict_function', self, unique=True) expression = lazy_function(*self.features) copy.add_virtual_column(self.prediction_label, expression, unique=False) return copy def fit(self, dataframe): ''' Fit the KMeans model to the dataframe. :param dataframe: A vaex DataFrame. ''' if self.init == 'random': rng = np.random.RandomState(self.random_state) self.run_cluster_centers = [ self.generate_cluster_centers_random(dataframe, rng) for k in range(self.n_init) ] else: if self.n_init > 1: print( "WARNING: n_init > 1 , but init given, only doing one run") self.run_cluster_centers = [self.init] done = [False] * len(self.run_cluster_centers) alldone = False first = True previous_inertias = None iteration = 0 inertias_list = [] while not alldone: new_centers, inertias = self._find_centers_and_inertias( dataframe, done) if self.verbose: inertia_msges = [('--' if d else '{: 3}'.format(k)) for k, d in zip(inertias, done)] if len(inertias) == 1: inertia_msg = inertia_msges[0] else: inertia_msg = " | ".join(inertia_msges) print('Iteration {: 4}, inertia {}'.format( iteration, inertia_msg)) if not first: # we can only to a check after the second iteration done = self._is_done(previous_inertias, inertias) alldone = np.all(done) else: first = False iteration += 1 if (iteration >= self.max_iter): logger_km.debug('reached max iterations: %s', self.max_iter) alldone = True previous_inertias = inertias inertias_list.append(inertias) self.run_cluster_centers = new_centers best_run = np.argmin(inertias) self.inertia = inertias[best_run] self.inertias = np.array(inertias_list)[:, best_run] self.cluster_centers = new_centers[best_run] def _is_done(self, inertias1, inertias2): diffs = [(inertia1 - inertia2) for inertia1, inertia2 in zip(inertias1, inertias2)] return [(diff < 1e-3) for diff in diffs] def _find_centers_and_inertias(self, dataframe, done): done = np.array(done, dtype=np.int8) centroids = np.array(self.run_cluster_centers) runs = centroids.shape[0] clusters = centroids.shape[1] dimensions = centroids.shape[2] # print("k =", k) assert dimensions == len( self.features ), "nr of dimensions for centroid should equal nr of features" def map(*blocks): # this will be called with a chunk of the data sumpos = np.zeros((runs, clusters, dimensions)) counts = np.zeros((runs, clusters)) inertia = np.zeros((runs)) if True: centroid_stats(centroids, counts, sumpos, inertia, *blocks) else: # this is the pure python code # although not made for multiple runs yet distances_sq = self._calculate_distances_squared(*blocks) classes = np.argmin(distances_sq, axis=0) distances_sq = np.choose(classes, distances_sq) inertia = (distances_sq).sum() for i in range(k): mask = classes == i counts[i] = np.sum(mask) for d in range(dimensions): sumpos[i, d] += blocks[d][mask].sum() return sumpos, counts, inertia def reduce(x, y): sumpos1, counts1, inertia1 = x sumpos2, counts2, inertia2 = y return sumpos1 + sumpos2, counts1 + counts2, inertia1 + inertia2 sumpos, counts, inertia = dataframe.map_reduce(map, reduce, self.features) means = (sumpos / counts[:, :, np.newaxis]) return means.tolist(), inertia
class _LinearBase(state.HasState): features = traitlets.List(traitlets.Unicode()) binned = traitlets.CBool(True) shape = traitlets.CInt(64) limits = traitlets.List(traitlets.List(traitlets.CFloat(), allow_none=True), allow_none=True).tag(output=True) fit_intercept = traitlets.CBool(True) coef_ = traitlets.Union([ traitlets.List(traitlets.CFloat()), traitlets.List(traitlets.List(traitlets.CFloat())) ]).tag(output=True) # intercept_ = traitlets.List(traitlets.CFloat).tag(output=True) intercept_ = traitlets.Union( [traitlets.CFloat(), traitlets.List(traitlets.CFloat())]).tag(output=True) _sk_params = traitlets.Any() prediction_name = traitlets.Unicode(default_value='linear_prediction') def transform(self, dataset): ds = dataset.copy() expression = self.coef_[0] * ds[self.features[0]] for coef, feature in zip(self.coef_, self.features[1:]): expression = expression + coef * ds[feature] expression = self.intercept_ + expression ds.add_virtual_column(self.prediction_name, expression, unique=False) return ds def fit(self, dataset, y_expression, progress=False): assert len(set(self.features)) == len( self.features), "duplicate features" if not self.binned: X = np.array(dataset[self.features]) y = dataset.evaluate(y_expression) m = self._make_model() m.fit(X, y) self.coef_ = m.coef_.tolist() self.intercept_ = m.intercept_.tolist() else: limits = self.limits if limits == []: limits = None binby = self.features + [y_expression] limits, shapes = dataset.limits(binby, limits, shape=self.shape) self.limits = listify(limits) counts = dataset.count(binby=binby, limits=limits, shape=shapes) mask = counts > 0 def coordinates(expression, limits, shape): if dataset.is_category(expression): return np.arange(dataset.category_count(expression)) else: return dataset.bin_centers(expression, limits, shape) centers = [ coordinates(expression, l, shape) for expression, l, shape in zip(binby, self.limits, shapes) ] # l = ds.bin_centers('y', limits[1], shape) centers = np.meshgrid(*centers, indexing='ij') centers = [c[mask] for c in centers] # m = lin.LinearRegression(fit_intercept=self.fit_intercept) m = self._make_model() X = np.array(centers[:-1]).reshape(-1, len(self.features)) y = centers[-1].reshape(-1) weights = counts[mask] m.fit(X, y, sample_weight=weights) self.coef_ = m.coef_.tolist() self.intercept_ = m.intercept_.tolist() # self._sk_params = m.get_params() self.last_model = m def predict(self, dataset): X = np.array(dataset[self.features]) return self.last_model.predict(X)
class Figure(ipywebrtc.MediaStream): """Widget class representing a volume (rendering) using three.js""" _view_name = Unicode('FigureView').tag(sync=True) _view_module = Unicode('ipyvolume').tag(sync=True) _model_name = Unicode('FigureModel').tag(sync=True) _model_module = Unicode('ipyvolume').tag(sync=True) _view_module_version = Unicode(semver_range_frontend).tag(sync=True) _model_module_version = Unicode(semver_range_frontend).tag(sync=True) volume_data = Array(default_value=None, allow_none=True).tag(sync=True, **array_cube_tile_serialization) volume_data_original = Array(default_value=None, allow_none=True) volume_data_max_shape = traitlets.CInt(None, allow_none=True) # TODO: allow this to be a list eye_separation = traitlets.CFloat(6.4).tag(sync=True) volume_data_min = traitlets.CFloat().tag(sync=True) volume_data_max = traitlets.CFloat().tag(sync=True) volume_show_min = traitlets.CFloat().tag(sync=True) volume_show_max = traitlets.CFloat().tag(sync=True) volume_clamp_min = traitlets.CBool(False).tag(sync=True) volume_clamp_max = traitlets.CBool(False).tag(sync=True) opacity_scale = traitlets.CFloat(1.0).tag(sync=True) tf = traitlets.Instance(TransferFunction, allow_none=True).tag(sync=True, **ipywidgets.widget_serialization) volume_rendering_method = traitlets.Enum(values=['NORMAL', 'MAX_INTENSITY'], default_value='NORMAL').tag(sync=True) volume_rendering_lighting = traitlets.Bool(True).tag(sync=True) scatters = traitlets.List(traitlets.Instance(Scatter), [], allow_none=False).tag(sync=True, **ipywidgets.widget_serialization) meshes = traitlets.List(traitlets.Instance(Mesh), [], allow_none=False).tag(sync=True, **ipywidgets.widget_serialization) animation = traitlets.Float(1000.0).tag(sync=True) animation_exponent = traitlets.Float(.5).tag(sync=True) ambient_coefficient = traitlets.Float(0.5).tag(sync=True) diffuse_coefficient = traitlets.Float(0.8).tag(sync=True) specular_coefficient = traitlets.Float(0.5).tag(sync=True) specular_exponent = traitlets.Float(5).tag(sync=True) stereo = traitlets.Bool(False).tag(sync=True) camera_control = traitlets.Unicode(default_value='trackball').tag(sync=True) camera_fov = traitlets.CFloat(45,min=0.1,max=179.9).tag(sync=True) camera_center = traitlets.List(traitlets.CFloat, default_value=[0, 0, 0]).tag(sync=True) #Tuple(traitlets.CFloat(0), traitlets.CFloat(0), traitlets.CFloat(0)).tag(sync=True) camera = traitlets.Instance(pythreejs.Camera).tag(sync=True, **ipywidgets.widget_serialization) @traitlets.default('camera') def _default_camera(self): # return pythreejs.CombinedCamera(fov=46, position=(0, 0, 2), width=400, height=500) return pythreejs.PerspectiveCamera(fov=46, position=(0, 0, 2), width=400, height=500) scene = traitlets.Instance(pythreejs.Scene).tag(sync=True, **ipywidgets.widget_serialization) @traitlets.default('scene') def _default_scene(self): # could be removed when https://github.com/jovyan/pythreejs/issues/176 is solved # the default for pythreejs is white, which leads the volume rendering pass to make everything white return pythreejs.Scene(background=None) width = traitlets.CInt(500).tag(sync=True) height = traitlets.CInt(400).tag(sync=True) downscale = traitlets.CInt(1).tag(sync=True) displayscale = traitlets.CFloat(1).tag(sync=True) capture_fps = traitlets.CFloat(None, allow_none=True).tag(sync=True) cube_resolution = traitlets.CInt(512).tag(sync=True) show = traitlets.Unicode("Volume").tag(sync=True) # for debugging xlim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True) ylim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True) zlim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True) extent = traitlets.Any().tag(sync=True) extent_original = traitlets.Any() matrix_projection = traitlets.List(traitlets.CFloat, default_value=[0] * 16, allow_none=True, minlen=16, maxlen=16).tag(sync=True) matrix_world = traitlets.List(traitlets.CFloat, default_value=[0] * 16, allow_none=True, minlen=16, maxlen=16).tag(sync=True) xlabel = traitlets.Unicode("x").tag(sync=True) ylabel = traitlets.Unicode("y").tag(sync=True) zlabel = traitlets.Unicode("z").tag(sync=True) style = traitlets.Dict(default_value=ipyvolume.styles.default).tag(sync=True) render_continuous = traitlets.Bool(False).tag(sync=True) selector = traitlets.Unicode(default_value='lasso').tag(sync=True) selection_mode = traitlets.Unicode(default_value='replace').tag(sync=True) mouse_mode = traitlets.Unicode(default_value='normal').tag(sync=True) panorama_mode = traitlets.Enum(values=['no', '360', '180'], default_value='no').tag(sync=True) #xlim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True) #y#lim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True) #zlim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True) def __init__(self, **kwargs): super(Figure, self).__init__(**kwargs) self._screenshot_handlers = widgets.CallbackDispatcher() self._selection_handlers = widgets.CallbackDispatcher() self.on_msg(self._handle_custom_msg) self._update_volume_data() self.observe(self.update_volume_data, ['xlim', 'ylim', 'zlim', 'volume_data_original', 'volume_data_max_shape']) @debounced(method=True) def update_volume_data(self, change=None): self._update_volume_data() def _update_volume_data(self): if self.volume_data_original is None: return if all([k <= self.volume_data_max_shape for k in self.volume_data_original.shape]): self.volume_data = self.volume_data_original self.extent = self.extent_original return shape = self.volume_data_original.shape ex = self.extent_original viewx, xt = grid_slice(ex[0][0], ex[0][1], shape[2], *self.xlim) viewy, yt = grid_slice(ex[1][0], ex[1][1], shape[1], *self.ylim) viewz, zt = grid_slice(ex[2][0], ex[2][1], shape[0], *self.zlim) view = [slice(*viewz), slice(*viewy), slice(*viewx)] data_view = self.volume_data_original[view] extent = [xt, yt, zt] data_view, extent = reduce_size(data_view, self.volume_data_max_shape, extent) self.volume_data = np.array(data_view) self.extent = extent def __enter__(self): """Sets this figure as the current in the pylab API Example: >>> f1 = ipv.figure(1) >>> f2 = ipv.figure(2) >>> with f1: >>> ipv.scatter(x, y, z) >>> assert ipv.gcf() is f2 """ import ipyvolume as ipv self._previous_figure = ipv.gcf() ipv.figure(self) def __exit__(self, type, value, traceback): import ipyvolume as ipv ipv.figure(self._previous_figure) del self._previous_figure def screenshot(self, width=None, height=None, mime_type='image/png'): self.send({'msg':'screenshot', 'width':width, 'height':height, 'mime_type':mime_type}) def on_screenshot(self, callback, remove=False): self._screenshot_handlers.register_callback(callback, remove=remove) def _handle_custom_msg(self, content, buffers): if content.get('event', '') == 'screenshot': self._screenshot_handlers(content['data']) elif content.get('event', '') == 'selection': self._selection_handlers(content['data']) def on_selection(self, callback, remove=False): self._selection_handlers.register_callback(callback, remove=remove) def project(self, x, y, z): W = np.matrix(self.matrix_world).reshape((4,4)) .T P = np.matrix(self.matrix_projection).reshape((4,4)).T M = np.dot(P, W) x = np.asarray(x) vertices = np.array([x, y, z, np.ones(x.shape)]) screen_h = np.tensordot(M, vertices, axes=(1, 0)) xy = screen_h[:2] / screen_h[3] return xy
class QMTSlopeTile(StrictHasTraits, object): name = "qmtslope" urltemplate = "https://assets.agi.com/stk-terrain/world/{z}/{x}/{y}.terrain?v=1.31376.0" geodetic = quantized_mesh_tile.global_geodetic.GlobalGeodetic(True) opacity = traitlets.CFloat(max=1.0, min=0.0, default_value=1.0) line = traitlets.CBool(default_value=False) @property def storage_key(self): key = self.name if self.line: key = key + "_line_True" return key @classmethod def spanning_tile_coords(cls, xyzp): mb = mercantile.bounds(xyzp) z = xyzp.z - 1 if mb.north >= 49.0 and z > 14: z = 14 elif z > 15: z = 15 spanning_tiles = set( cls.geodetic.LonLatToTile(lat, lon, z) for lat, lon in itertools.product((mb.west, mb.east), (mb.north, mb.south))) return [(x, y, z) for x, y in spanning_tiles] @classmethod def load_qmt(cls, x, y, z, content): cls.geodetic.TileBounds(x, y, z) bounds = dict( zip(("west", "south", "east", "north"), cls.geodetic.TileBounds(x, y, z))) tile = quantized_mesh_tile.TerrainTile(**bounds) tile.fromStringIO(io.BytesIO(content)) return tile @classmethod def vnorm(self, vectors): #No axis arg in numpy 1.6 norm return sum([vectors[..., i]**2 for i in (0, 1, 2)])**.5 @classmethod def triangle_slope_angles(cls, qt_coords, qt_triangles): qt_meters = qt_coords.copy() qt_meters[..., 0], qt_meters[..., 1] = mercantile.xy(qt_coords[..., 0], qt_coords[..., 1]) tri_crosses = numpy.cross( qt_meters[qt_triangles[..., 0]] - qt_meters[qt_triangles[..., 1]], qt_meters[qt_triangles[..., 0]] - qt_meters[qt_triangles[..., 2]]) off_z_angle = numpy.rad2deg( numpy.arccos(tri_crosses[:, 2] / cls.vnorm(tri_crosses))) return off_z_angle @ndb.tasklet def fetch_tile_data(self, x, y, z): context = ndb.get_context() tile_data = yield context.urlfetch( self.urltemplate.format(x=x, y=y, z=z)) if tile_data.status_code != 200: logging.error("error fetching qmt: %r url: %s result: %s", (x, y, z), self.urltemplate.format(x=x, y=y, z=z), result) raise ValueError("error fetching qmt: %r url: %s result: %s", (x, y, z), self.urltemplate.format(x=x, y=y, z=z), result) logging.info("loading qmt file: %s", ((x, y, z), )) qmt = self.load_qmt(x, y, z, tile_data.content) logging.info("generating mesh array: %s", ((x, y, z), )) raise ndb.Return({ "coords": numpy.array(qmt.getVerticesCoordinates()), "triangles": numpy.array(numpy.array(qmt.indices)) }) @ndb.tasklet def render_async(self, tile): tile = mercantile.Tile(**tile) cache = memcache._CLIENT targets = self.spanning_tile_coords(tile) tkeys = {str(t): t for t in targets} cached_coords = yield cache.get_multi_async( tkeys, key_prefix="%s+tilearray+" % self.name) newkeys = set(tkeys) - set(cached_coords) newdata = yield tuple(self.fetch_tile_data(*tkeys[c]) for c in newkeys) new = dict(zip(newkeys, newdata)) yield cache.set_multi_async(new, key_prefix="%s+tilearray+" % self.name) qm_coords = [] qm_triangles = [] ind = 0 for qmt in list(cached_coords.values()) + list(new.values()): qm_coords.append(qmt["coords"]) qm_triangles.append(qmt["triangles"] + ind) ind += len(qm_coords[-1]) qm_coords = numpy.concatenate(qm_coords, axis=0) qm_triangles = numpy.concatenate(qm_triangles, axis=0).reshape((-1, 3)) raise ndb.Return(self._render_from_qmt(tile, qm_coords, qm_triangles)) def render(self, tile): tile = mercantile.Tile(**tile) qmt_data = {(x, y, z): requests.get(self.urltemplate.format(x=x, y=y, z=z)).content for x, y, z in self.spanning_tile_coords(tile)} logging.info("loading qmt files: %s", qmt_data.keys()) qmts = [self.load_qmt(x, y, z, d) for (x, y, z), d in qmt_data.items()] logging.info("generating mesh arrays") qm_coords = [] qm_triangles = [] ind = 0 for qmt in qmts: qm_coords.append(numpy.array(qmt.getVerticesCoordinates())) qm_triangles.append(numpy.array(qmt.indices) + ind) ind += len(qm_coords[-1]) qm_coords = numpy.concatenate(qm_coords, axis=0) qm_triangles = numpy.concatenate(qm_triangles, axis=0).reshape((-1, 3)) return self._render_from_qmt(tile, qm_coords, qm_triangles) def _render_from_qmt(self, tile, qm_coords, qm_triangles): logging.info("calculating slope angles: %i", len(qm_triangles)) slope_angles = self.triangle_slope_angles(qm_coords, qm_triangles) logging.info("coloring slope") mb = mercantile.bounds(tile) qm_xs = (qm_coords[..., 0] - mb.west) / ((mb.east - mb.west) / 255) qm_ys = (qm_coords[..., 1] - mb.north) / ((mb.south - mb.north) / 255) qm_colors = angle_to_rbga(slope_angles) qm_pix = numpy.empty((len(qm_coords), 2), dtype=int) qm_pix[..., 0] = qm_xs + 128 qm_pix[..., 1] = qm_ys + 128 logging.info("rendering tile") rbuff = Image.new("RGBA", (256 * 2, 256 * 2)) rdraw = ImageDraw.ImageDraw(rbuff) for ti in range(len(qm_triangles)): rdraw.polygon(map(tuple, qm_pix[qm_triangles[ti]]), fill=tuple(qm_colors[ti]), outline=(0, 0, 0, 255) if self.line else None) res = rbuff.crop((128, 128, 255 + 128, 255 + 128)) return res
class PCA(Transformer): '''Transform a set of features using a Principal Component Analysis. Example: >>> import vaex >>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10]) >>> df # x y 0 2 -2 1 5 3 2 7 0 3 2 0 4 15 10 >>> pca = vaex.ml.PCA(n_components=2, features=['x', 'y']) >>> pca.fit_transform(df) # x y PCA_0 PCA_1 0 2 -2 5.92532 0.413011 1 5 3 0.380494 -1.39112 2 7 0 0.840049 2.18502 3 2 0 4.61287 -1.09612 4 15 10 -11.7587 -0.110794 ''' # title = traitlets.Unicode(default_value='PCA', read_only=True).tag(ui='HTML') n_components = traitlets.Int( help= 'Number of components to retain. If None, all the components will be retained.' ).tag(ui='IntText') prefix = traitlets.Unicode(default_value="PCA_", help=help_prefix) progress = traitlets.CBool( default_value=False, help='If True, display a progressbar of the PCA fitting process.').tag( ui='Checkbox') eigen_vectors_ = traitlets.List( traitlets.List(traitlets.CFloat()), help='The eigen vectors corresponding to each feature').tag( output=True) eigen_values_ = traitlets.List( traitlets.CFloat(), help='The eigen values that correspond to each feature.').tag( output=True) means_ = traitlets.List(traitlets.CFloat(), help='The mean of each feature').tag(output=True) @traitlets.default('n_components') def get_n_components_default(self): return len(self.features) def fit(self, df): '''Fit the PCA model to the DataFrame. :param df: A vaex DataFrame. ''' self.n_components = self.n_components or len(self.features) assert self.n_components >= 2, 'At least two features are required.' assert self.n_components <= len( self.features), 'Can not have more components than features.' C = df.cov(self.features, progress=self.progress) eigen_values, eigen_vectors = np.linalg.eigh(C) indices = np.argsort(eigen_values)[::-1] self.means_ = df.mean(self.features, progress=self.progress).tolist() self.eigen_vectors_ = eigen_vectors[:, indices].tolist() self.eigen_values_ = eigen_values[indices].tolist() def transform(self, df, n_components=None): '''Apply the PCA transformation to the DataFrame. :param df: A vaex DataFrame. :param n_components: The number of PCA components to retain. :return copy: A shallow copy of the DataFrame that includes the PCA components. :rtype: DataFrame ''' n_components = n_components or self.n_components copy = df.copy() name_prefix_offset = 0 eigen_vectors = np.array(self.eigen_vectors_) while self.prefix + str(name_prefix_offset) in copy.get_column_names( virtual=True, strings=True): name_prefix_offset += 1 expressions = [ copy[feature] - mean for feature, mean in zip(self.features, self.means_) ] for i in range(n_components): v = eigen_vectors[:, i] expr = dot_product(expressions, v) name = self.prefix + str(i + name_prefix_offset) copy[name] = expr return copy
class RobustScaler(Transformer): ''' The RobustScaler removes the median and scales the data according to a given percentile range. By default, the scaling is done between the 25th and the 75th percentile. Centering and scaling happens independently for each feature (column). Example: >>> import vaex >>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10]) >>> df # x y 0 2 -2 1 5 3 2 7 0 3 2 0 4 15 10 >>> scaler = vaex.ml.MaxAbsScaler(features=['x', 'y']) >>> scaler.fit_transform(df) # x y robust_scaled_x robust_scaled_y 0 2 -2 -0.333686 -0.266302 1 5 3 -0.000596934 0.399453 2 7 0 0.221462 0 3 2 0 -0.333686 0 4 15 10 1.1097 1.33151 ''' with_centering = traitlets.CBool( default_value=True, help='If True, remove the median.').tag(ui='Checkbox') with_scaling = traitlets.CBool( default_value=True, help= 'If True, scale each feature between the specified percentile range.' ).tag(ui='Checkbox') percentile_range = traitlets.Tuple( default_value=(25, 75), help='The percentile range to which to scale each feature to.').tag( ).tag(ui='FloatRangeSlider') prefix = traitlets.Unicode(default_value="robust_scaled_", help=help_prefix).tag(ui='Text') center_ = traitlets.List( traitlets.CFloat(), default_value=None, help='The median of each feature.').tag(output=True) scale_ = traitlets.List( traitlets.CFloat(), default_value=None, help='The percentile range for each feature.').tag(output=True) def fit(self, df): ''' Fit RobustScaler to the DataFrame. :param df: A vaex DataFrame. ''' # check the quantile range q_min, q_max = self.percentile_range if not 0 <= q_min <= q_max <= 100: raise ValueError('Invalid percentile range: %s' % (str(self.percentile_range))) if self.with_centering: self.center_ = df.percentile_approx(expression=self.features, percentage=50).tolist() if self.with_scaling: self.scale_ = (df.percentile_approx(expression=self.features, percentage=q_max) - df.percentile_approx(expression=self.features, percentage=q_min)).tolist() def transform(self, df): ''' Transform a DataFrame with a fitted RobustScaler. :param df: A vaex DataFrame. :returns copy: a shallow copy of the DataFrame that includes the scaled features. :rtype: DataFrame ''' copy = df.copy() for i, feature in enumerate(self.features): name = self.prefix + feature expr = copy[feature] if self.with_centering: expr -= self.center_[i] if self.with_scaling: expr /= self.scale_[i] copy[name] = expr return copy
class Generator(t.HasTraits): '''Generator Model''' name = t.CUnicode(default_value='GenCo0', help='Name of Generator (str)') generator_bus = t.CUnicode(default_value='Bus0', help='Bus of Generator (str)') generator_voltage = t.CFloat( default_value=1.0, help='Nominal voltage of the generator (p.u.)') base_power = t.CFloat(default_value=100.0, help='Base power of the generator (MVA)') generation_type = t.Enum(['COAL', 'NATURALGAS', 'WIND'], default_value='COAL') minimum_up_time = t.CInt(default_value=0, min=0, help='Minimum up time (hrs)') minimum_down_time = t.CInt(default_value=0, min=0, help='Minimum down time (hrs)') ramp_up_rate = t.CFloat(default_value=0, min=0, help='Ramp up rate (MW/hr)') ramp_down_rate = t.CFloat(default_value=0, min=0, help='Ramp down rate (MW/hr)') maximum_real_power = t.CFloat(default_value=0, min=0, help='Capacity of Generator (MW)') minimum_real_power = t.CFloat(default_value=0, min=0, help='Minimum generation (MW)') maximum_imag_power = t.CFloat(default_value=0, help='Maximum reactive generation (MVAR)') minimum_imag_power = t.CFloat(default_value=0, help='Minimum reactive generation (MVAR)') initial_real_power = t.CFloat(default_value=0, min=0, help='Initial power generation (MW)') initial_imag_power = t.CFloat(default_value=0, min=0, help='Initial power generation (MVAR)') initial_status = t.CBool(default_value=True, min=0, help='Initial status (bool)') startup_time = t.CInt(default_value=0, min=0, help='Startup time (hrs)') shutdown_time = t.CInt(default_value=0, min=0, help='Shutdown time (hrs)') nsegments = t.CInt(default_value=2, min=MINIMUM_COST_CURVE_SEGMENTS, max=MAXIMUM_COST_CURVE_SEGMENTS, help='Number of data points for piecewise linear') cost_curve_points = tt.Array(default_value=[0, 0, 0], minlen=(MINIMUM_COST_CURVE_SEGMENTS + 1), maxlen=(MAXIMUM_COST_CURVE_SEGMENTS + 1)) cost_curve_values = tt.Array(default_value=[0, 0, 0], minlen=(MINIMUM_COST_CURVE_SEGMENTS + 1), maxlen=(MAXIMUM_COST_CURVE_SEGMENTS + 1)) noload_cost = t.CFloat(default_value=0, min=0, help='No-Load Cost of a Generator ($/hr)') startup_cost = t.CFloat(default_value=0, min=0, help='Startup Cost of a Generator ($/hr)') inertia = t.CFloat(allow_none=True, default_value=None, min=0, help='Inertia of generator (NotImplemented)') droop = t.CFloat(allow_none=True, default_value=None, min=0, help='Droop of generator (NotImplemented)') def __init__(self, *args, **kwargs): super(Generator, self).__init__(*args, **kwargs) @property def _npoints(self): return self.nsegments + 1 @property def ramp_rate(self): raise AttributeError( "'{class_name}' object has no attribute 'ramp_rate'. Try 'ramp_up_rate' or 'ramp_down_rate'." .format(class_name=self.__class__.__name__)) @ramp_rate.setter def ramp_rate(self, v): self.ramp_up_rate = v self.ramp_down_rate = v @t.observe('noload_cost') def _callback_noload_cost_update_points_values(self, change): self.cost_curve_values = [change['new']] * self._npoints return change['new'] @t.observe('minimum_real_power') def _callback_minimum_real_power_update_points_values(self, change): self.cost_curve_points = np.linspace(change['new'], self.maximum_real_power, self._npoints) return change['new'] @t.observe('maximum_real_power') def _callback_maximum_real_power_update_points_values(self, change): self.cost_curve_points = np.linspace(self.minimum_real_power, change['new'], self._npoints) self.ramp_rate = self.maximum_real_power return change['new'] @t.observe('nsegments') def _callback_nsegments_update_points_values(self, change): self.cost_curve_points = np.linspace(self.minimum_real_power, self.maximum_real_power, change['new'] + 1) self.cost_curve_values = [self.noload_cost] * (change['new'] + 1) return change['new'] @t.validate('cost_curve_points', 'cost_curve_values') def _validate_max_length(self, proposal): if not len(proposal['value']) == self._npoints: raise t.TraitError( 'len({class_name}().{trait_name}) must be equal to {class_name}().nsegments + 1. Proposed {trait_name} is {proposal}' .format(class_name=proposal['owner'].__class__.__name__, trait_name=proposal['trait'].name, proposal=proposal['value'])) return proposal['value'] @t.validate('ramp_up_rate', 'ramp_down_rate', 'initial_real_power', 'initial_imag_power') def _less_than_maximum_real_power_check(self, proposal): if not proposal['value'] <= self.maximum_real_power: raise t.TraitError( '{class_name}().{trait_name} must be a less than or equal to {class_name}().maximum_real_power.' .format(class_name=proposal['owner'].__class__.__name__, trait_name=proposal['trait'].name)) else: return proposal['value']