class AnimationWidget(widgets.DOMWidget):
"""
A widget that periodic increment a value
:param value: A float between 0 and 1
:param run: boolean with the state of the timer. True, the timer is enable
Produces the following signal. A sampling rate, the value is interpolated
with the equation val = 1/Period * t
1- ^ ____
| /
|/
0- |----->
|
period
"""
_view_name = traitlets.Unicode('AnimationView').tag(sync=True)
_model_name = traitlets.Unicode('AnimationModel').tag(sync=True)
_view_module = traitlets.Unicode('animation-widget').tag(sync=True)
_model_module = traitlets.Unicode('animation-widget').tag(sync=True)
# Signal value
value = traitlets.CFloat(0.0).tag(sync=True)
# Boolean timer is active
run = traitlets.CBool(False).tag(sync=True)
# Signal period (in ms)
period = traitlets.CFloat(5000).tag(sync=True)
# Number of samples in period
nbsamples = traitlets.CInt(100).tag(sync=True)
# Loop
loop = traitlets.CBool(False).tag(sync=True)
class Scatter(widgets.Widget):
_view_name = Unicode('ScatterView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('ScatterModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True, **array_sequence_serialization)
y = Array(default_value=None).tag(sync=True, **array_sequence_serialization)
z = Array(default_value=None).tag(sync=True, **array_sequence_serialization)
vx = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
vy = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
vz = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
size = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization),
traitlets.Float().tag(sync=True)],
default_value=5).tag(sync=True)
size_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization),
traitlets.Float().tag(sync=True)],
default_value=7).tag(sync=True)
color = Array(default_value="red", allow_none=True).tag(sync=True, **color_serialization)
color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization),
Unicode().tag(sync=True)],
default_value="green").tag(sync=True)
geo = traitlets.Unicode('diamond').tag(sync=True)
connected = traitlets.CBool(default_value=False).tag(sync=True)
visible = traitlets.CBool(default_value=True).tag(sync=True)
texture = traitlets.Union([
traitlets.Instance(ipywebrtc.MediaStream),
Unicode(),
traitlets.List(Unicode, [], allow_none=True),
Image(default_value=None, allow_none=True),
traitlets.List(Image(default_value=None, allow_none=True))
]).tag(sync=True, **texture_serialization)
material = traitlets.Instance(pythreejs.ShaderMaterial, help='A :any:`pythreejs.ShaderMaterial` that is used for the mesh')\
.tag(sync=True, **ipywidgets.widget_serialization)
@traitlets.default('material')
def _default_material(self):
return pythreejs.ShaderMaterial()
line_material = traitlets.Instance(pythreejs.ShaderMaterial, help='A :any:`pythreejs.ShaderMaterial` that is used for the lines/wireframe')\
.tag(sync=True, **ipywidgets.widget_serialization)
@traitlets.default('line_material')
def _default_line_material(self):
return pythreejs.ShaderMaterial()
class Scatter(widgets.DOMWidget):
_view_name = Unicode('ScatterView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('ScatterModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
y = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
z = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
vx = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
vy = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
vz = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
selected = Array(default_value=None,
allow_none=True).tag(sync=True,
**array_sequence_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
size = traitlets.Union([
Array(default_value=None, allow_none=True).tag(
sync=True, **array_sequence_serialization),
traitlets.Float().tag(sync=True)
],
default_value=5).tag(sync=True)
size_selected = traitlets.Union([
Array(default_value=None, allow_none=True).tag(
sync=True, **array_sequence_serialization),
traitlets.Float().tag(sync=True)
],
default_value=7).tag(sync=True)
color = Array(default_value="red",
allow_none=True).tag(sync=True, **color_serialization)
color_selected = traitlets.Union([
Array(default_value=None, allow_none=True).tag(sync=True,
**color_serialization),
Unicode().tag(sync=True)
],
default_value="green").tag(sync=True)
geo = traitlets.Unicode('diamond').tag(sync=True)
connected = traitlets.CBool(default_value=False).tag(sync=True)
visible = traitlets.CBool(default_value=True).tag(sync=True)
visible_lines = traitlets.CBool(default_value=False).tag(sync=True)
visible_markers = traitlets.CBool(default_value=True).tag(sync=True)
class Mesh(widgets.DOMWidget):
_view_name = Unicode('MeshView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('MeshModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
y = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
z = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
u = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
v = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
triangles = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
lines = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
texture = Image(default_value=None,
allow_none=True).tag(sync=True, **image_serialization)
# selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
color = Array(default_value="red",
allow_none=True).tag(sync=True, **color_serialization)
# color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization),
# Unicode().tag(sync=True)],
# default_value="green").tag(sync=True)
# geo = traitlets.Unicode('diamond').tag(sync=True)
wire = traitlets.CBool(default_value=False).tag(sync=True)
class Mesh(widgets.DOMWidget):
_view_name = Unicode('MeshView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('MeshModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
y = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
z = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
u = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
v = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
triangles = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
lines = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
texture = traitlets.Union([
traitlets.Instance(ipywebrtc.MediaStream),
Unicode(),
traitlets.List(Unicode, [], allow_none=True),
Image(default_value=None, allow_none=True),
traitlets.List(Image(default_value=None, allow_none=True))
]).tag(sync=True, **texture_serialization)
# selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
color = Array(default_value="red",
allow_none=True).tag(sync=True, **color_serialization)
# color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization),
# Unicode().tag(sync=True)],
# default_value="green").tag(sync=True)
# geo = traitlets.Unicode('diamond').tag(sync=True)
visible = traitlets.CBool(default_value=True).tag(sync=True)
visible_lines = traitlets.CBool(default_value=True).tag(sync=True)
visible_faces = traitlets.CBool(default_value=True).tag(sync=True)
side = traitlets.CaselessStrEnum(['front', 'back', 'both'],
'both').tag(sync=True)
class Mesh(widgets.Widget):
_view_name = Unicode('MeshView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('MeshModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
y = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
z = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
u = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
v = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
triangles = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
lines = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
texture = traitlets.Union([
traitlets.Instance(ipywebrtc.MediaStream),
Unicode(),
traitlets.List(Unicode, [], allow_none=True),
Image(default_value=None, allow_none=True),
traitlets.List(Image(default_value=None, allow_none=True)),
]).tag(sync=True, **texture_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
color = Array(default_value="red",
allow_none=True).tag(sync=True, **color_serialization)
visible = traitlets.CBool(default_value=True).tag(sync=True)
material = traitlets.Instance(
pythreejs.ShaderMaterial,
help='A :any:`pythreejs.ShaderMaterial` that is used for the mesh'
).tag(sync=True, **widgets.widget_serialization)
@traitlets.default('material')
def _default_material(self):
return pythreejs.ShaderMaterial(side=pythreejs.enums.Side.DoubleSide)
line_material = traitlets.Instance(
pythreejs.ShaderMaterial,
help=
'A :any:`pythreejs.ShaderMaterial` that is used for the lines/wireframe'
).tag(sync=True, **widgets.widget_serialization)
@traitlets.default('line_material')
def _default_line_material(self):
return pythreejs.ShaderMaterial()
class Mesh(widgets.DOMWidget):
_view_name = Unicode('MeshView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('MeshModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
x = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
y = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
z = Array(default_value=None).tag(sync=True,
**array_sequence_serialization)
u = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
v = Array(default_value=None,
allow_none=True).tag(sync=True, **array_sequence_serialization)
triangles = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
lines = Array(default_value=None,
allow_none=True).tag(sync=True, **array_serialization)
texture = traitlets.Union([
traitlets.Instance(ipywebrtc.MediaStream),
Unicode(),
traitlets.List(Unicode, [], allow_none=True),
Image(default_value=None, allow_none=True),
traitlets.List(Image(default_value=None, allow_none=True))
]).tag(sync=True, **texture_serialization)
# selected = Array(default_value=None, allow_none=True).tag(sync=True, **array_sequence_serialization)
sequence_index = Integer(default_value=0).tag(sync=True)
color = Array(default_value="red",
allow_none=True).tag(sync=True, **color_serialization)
# color_selected = traitlets.Union([Array(default_value=None, allow_none=True).tag(sync=True, **color_serialization),
# Unicode().tag(sync=True)],
# default_value="green").tag(sync=True)
# geo = traitlets.Unicode('diamond').tag(sync=True)
visible = traitlets.CBool(default_value=True).tag(sync=True)
material = traitlets.Instance(pythreejs.ShaderMaterial).tag(
sync=True, **ipywidgets.widget_serialization)
@traitlets.default('material')
def _default_material(self):
return pythreejs.ShaderMaterial(side=pythreejs.Side.DoubleSide)
line_material = traitlets.Instance(pythreejs.ShaderMaterial).tag(
sync=True, **ipywidgets.widget_serialization)
@traitlets.default('line_material')
def _default_line_material(self):
return pythreejs.ShaderMaterial()
class Branch(t.HasTraits):
"Branch model"
name = t.CUnicode(default_value='Branch1', help='Name of Branch (str)')
from_bus = t.CUnicode(default_value='Bus1', help='Name of From Bus')
to_bus = t.CUnicode(default_value='Bus2', help='Name of To Bus')
resistance = t.CFloat(default_value=0, help='Branch resistance (p.u.)')
reactance = t.CFloat(default_value=0, help='Branch reactance (p.u.)')
susceptance = t.CFloat(default_value=0, help='Branch susceptance (p.u.)')
rating = t.CFloat(default_value=M, help='Branch Rating')
status = t.CBool(default_value=1, help='Branch status')
angle_minimum = t.CFloat(default_value=0.0, help='Branch angle minimum')
angle_maximum = t.CFloat(default_value=0.0, help='Branch angle maximum')
tap = t.CFloat(default_value=None, allow_none=True)
shift = t.CFloat(default_value=None, allow_none=True)
from_real_power_flow = t.CFloat(default_value=None, allow_none=True, help='From active power flow (MW)')
from_imag_power_flow = t.CFloat(default_value=None, allow_none=True, help='From reactive power flow (MVAR)')
to_real_power_flow = t.CFloat(default_value=None, allow_none=True, help='To active power flow (MW)')
to_imag_power_flow = t.CFloat(default_value=None, allow_none=True, help='To reactive power flow (MVAR)')
class StandardScaler(Transformer):
'''Standardize features by removing thir mean and scaling them to unit variance.
Example:
>>> import vaex
>>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10])
>>> df
# x y
0 2 -2
1 5 3
2 7 0
3 2 0
4 15 10
>>> scaler = vaex.ml.StandardScaler(features=['x', 'y'])
>>> scaler.fit_transform(df)
# x y standard_scaled_x standard_scaled_y
0 2 -2 -0.876523 -0.996616
1 5 3 -0.250435 0.189832
2 7 0 0.166957 -0.522037
3 2 0 -0.876523 -0.522037
4 15 10 1.83652 1.85086
'''
# title = Unicode(default_value='Standard Scaler', read_only=True).tag(ui='HTML')
prefix = traitlets.Unicode(default_value="standard_scaled_",
help=help_prefix).tag(ui='Text')
with_mean = traitlets.CBool(
default_value=True,
help='If True, remove the mean from each feature.').tag(ui='Checkbox')
with_std = traitlets.CBool(
default_value=True,
help='If True, scale each feature to unit variance.').tag(
ui='Checkbox')
mean_ = traitlets.List(traitlets.CFloat(),
help='The mean of each feature').tag(output=True)
std_ = traitlets.List(
traitlets.CFloat(),
help='The standard deviation of each feature.').tag(output=True)
def fit(self, df):
'''
Fit StandardScaler to the DataFrame.
:param df: A vaex DataFrame.
'''
mean = df.mean(self.features, delay=True)
std = df.std(self.features, delay=True)
@vaex.delayed
def assign(mean, std):
self.mean_ = mean.tolist()
self.std_ = std.tolist()
assign(mean, std)
df.execute()
def transform(self, df):
'''
Transform a DataFrame with a fitted StandardScaler.
:param df: A vaex DataFrame.
:returns copy: a shallow copy of the DataFrame that includes the scaled features.
:rtype: DataFrame
'''
copy = df.copy()
for i, feature in enumerate(self.features):
name = self.prefix + feature
expression = copy[feature]
if self.with_mean:
expression -= self.mean_[i]
if self.with_std:
expression /= self.std_[i]
copy[name] = expression
return copy
class Plot2dSliced(PlotBase):
z = traitlets.Unicode(allow_none=False).tag(sync=True)
z_slice = traitlets.CInt(default_value=0).tag(
sync=True) #.tag(sync=True) # TODO: do linking at python side
z_shape = traitlets.CInt(default_value=10).tag(sync=True)
z_relative = traitlets.CBool(False).tag(sync=True)
z_min = traitlets.CFloat(default_value=None,
allow_none=True).tag(sync=True) #.tag(sync=True)
z_max = traitlets.CFloat(default_value=None,
allow_none=True).tag(sync=True) #.tag(sync=True)
def __init__(self, **kwargs):
self.z_min_extreme, self.z_max_extreme = kwargs["dataset"].minmax(
kwargs["z"])
super(Plot2dSliced, self).__init__(**kwargs)
self.create_tools()
def get_limits(self, limits):
limits = self.dataset.limits(self.get_binby(), limits)
limits = list([list(k) for k in limits])
if self.z_min is None:
self.z_min = limits[2][0]
if self.z_max is None:
self.z_max = limits[2][1]
limits[2][0] = self.z_min
limits[2][1] = self.z_max
return limits
def select_rectangle(self, x1, y1, x2, y2, mode="replace"):
dz = self.z_max - self.z_min
z1 = self.z_min + dz * self.z_slice / self.z_shape
z2 = self.z_min + dz * (self.z_slice + 1) / self.z_shape
spaces = [self.x, self.y, self.z]
limits = [[x1, x2], [y1, y2], [z1, z2]]
self.dataset.select_box(spaces, limits=limits, mode=mode)
def select_lasso(self, x, y, mode="replace"):
raise NotImplementedError("todo")
def get_grid(self):
zslice = self.grid[..., self.z_slice]
if self.z_relative:
with np.errstate(divide='ignore', invalid='ignore'):
zslice = zslice / self.grid.sum(axis=-1)
return zslice
#return self.grid[...,self.z_slice]
def get_vgrids(self):
def zsliced(grid):
return grid[..., self.z_slice] if grid is not None else None
return [
zsliced(grid) for grid in super(Plot2dSliced, self).get_vgrids()
]
def create_tools(self):
#super(Plot2dSliced, self).create_tools()
self.z_slice_slider = widgets.IntSlider(value=self.z_slice,
min=0,
max=self.z_shape - 1)
#self.add_control_widget(self.z_slice_slider)
self.z_slice_slider.observe(self._z_slice_changed, "value")
self.observe(self._z_slice_changed, "z_slice")
dz = self.z_max_extreme - self.z_min_extreme
self.z_range_slider = widgets.FloatRangeSlider(
min=min(self.z_min, self.z_min_extreme),
value=[self.z_min, self.z_max],
max=max(self.z_max, self.z_max_extreme),
step=dz / 1000)
self.z_range_slider.observe(self._z_range_changed_, names=["value"])
#self.observe(self.z_range_slider, "z_min")
self.z_control = widgets.VBox(
[self.z_slice_slider, self.z_range_slider])
self.add_control_widget(self.z_control)
def _z_range_changed_(self, changes, **kwargs):
#print("changes1", changes, repr(changes), kwargs)
self.limits[2][0], self.limits[2][1] =\
self.z_min, self.z_max = self.z_range_slider.value = changes["new"]
self.update_grid()
def _z_slice_changed(self, changes):
self.z_slice = self.z_slice_slider.value = changes["new"]
self._update_image()
def get_shape(self):
return vaex.dataset._expand_shape(self.shape, 2) + (self.z_shape, )
def get_vshape(self):
return vaex.dataset._expand_shape(self.vshape, 2) + (self.z_shape, )
def get_binby(self):
return [self.x, self.y, self.z]
class Volume(widgets.Widget):
"""Widget class representing a volume (rendering) using three.js."""
_view_name = Unicode('VolumeView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('VolumeModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
data = Array(default_value=None, allow_none=True).tag(sync=True, **array_cube_tile_serialization)
data_original = Array(default_value=None, allow_none=True)
data_max_shape = traitlets.CInt(None, allow_none=True) # TODO: allow this to be a list
data_min = traitlets.CFloat(0).tag(sync=True)
data_max = traitlets.CFloat(1).tag(sync=True)
show_min = traitlets.CFloat(0).tag(sync=True)
show_max = traitlets.CFloat(1).tag(sync=True)
clamp_min = traitlets.CBool(False).tag(sync=True)
clamp_max = traitlets.CBool(False).tag(sync=True)
opacity_scale = traitlets.CFloat(1.0).tag(sync=True)
brightness = traitlets.CFloat(1.0).tag(sync=True)
tf = traitlets.Instance(TransferFunction, allow_none=True).tag(sync=True, **widgets.widget_serialization)
ray_steps = traitlets.CInt(
None,
allow_none=True,
help='defines the length of the ray (1/ray_steps) for each step, in normalized coordintes.',
).tag(sync=True)
rendering_method = traitlets.Enum(values=['NORMAL', 'MAX_INTENSITY'], default_value='NORMAL').tag(sync=True)
lighting = traitlets.Bool(True).tag(sync=True)
extent = traitlets.Any().tag(sync=True)
extent_original = traitlets.Any()
def __init__(self, **kwargs):
super(Volume, self).__init__(**kwargs)
self._update_data()
self.observe(self.update_data, ['data_original', 'data_max_shape'])
def _listen_to(self, fig):
fig.observe(self.update_data, ['xlim', 'ylim', 'zlim'])
@debounced(method=True)
def update_data(self, change=None):
self._update_data()
def _update_data(self):
if self.data_original is None:
return
if all([k <= self.data_max_shape for k in self.data_original.shape]):
self.data = self.data_original
self.extent = self.extent_original
return
current_figure = ipv.gcf()
xlim = current_figure.xlim
ylim = current_figure.ylim
zlim = current_figure.zlim
shape = self.data_original.shape
ex = self.extent_original
viewx, xt = grid_slice(ex[0][0], ex[0][1], shape[2], *xlim)
viewy, yt = grid_slice(ex[1][0], ex[1][1], shape[1], *ylim)
viewz, zt = grid_slice(ex[2][0], ex[2][1], shape[0], *zlim)
view = [slice(*viewz), slice(*viewy), slice(*viewx)]
data_view = self.data_original[view]
extent = [xt, yt, zt]
data_view, extent = reduce_size(data_view, self.data_max_shape, extent)
self.data = np.array(data_view)
self.extent = extent
class Watchdog(widgets.Widget):
path = traitlets.CUnicode(".", sync=True)
description = traitlets.CUnicode(sync=True)
value = traitlets.Any(sync=True)
recursive = traitlets.CBool(False, sync=True)
watching = traitlets.CBool(False, sync=True)
poll_interval = traitlets.CInt(1, sync=True)
last_event = traitlets.Dict(sync=True)
file = traitlets.CUnicode(sync=True)
def __init__(self, *args, **kwargs):
super(Watchdog, self).__init__(*args, **kwargs)
self._wd_handlers = widgets.CallbackDispatcher()
self._wd_observer = None
self.on_msg(self._handle_wd_msg)
def __del__(self):
self.stop()
super(Watchdog, self).__del__()
def _handle_wd_msg(self, _, content, buffers):
""" Be very careful here: inspecting the messages in the
browser may be the only way to find problems
"""
self.last_event = content
event = content.get('event', None)
if event is None:
return
if (not self.file) or fnmatch(content["src_path"], self.file):
self._wd_handlers(self, content)
def on_any(self, callback, remove=False):
self._wd_handlers.register_callback(callback, remove=remove)
def ls(self):
return glob.glob(os.path.join(self.path, self.file or "*.*"))
def touch(self, idx=0):
os.utime(self.ls()[idx], None)
def start(self):
self.stop()
ip = get_ipython()
self._wd_observer = subprocess.Popen(map(str, [
sys.executable, "-m", "ipywatchdogwidget.observe", "--model-id",
self.model_id, "--path",
os.path.abspath(self.path), "--poll-interval", self.poll_interval,
"--port", ip.kernel._recorded_ports["shell"], "--recursive",
self.recursive, "--session-key",
ip.kernel.session.key.decode()
]),
stderr=subprocess.STDOUT)
self.watching = True
def stop(self):
if self._wd_observer:
parent = psutil.Process(self._wd_observer.pid)
[child.kill() for child in parent.children(recursive=True)]
parent.kill()
self._wd_observer = None
self.watching = False
class KMeans(vaex.ml.state.HasState):
'''The KMeans clustering algorithm.
Example:
>>> import vaex.ml
>>> import vaex.ml.cluster
>>> df = vaex.ml.datasets.load_iris()
>>> features = ['sepal_width', 'petal_length', 'sepal_length', 'petal_width']
>>> cls = vaex.ml.cluster.KMeans(n_clusters=3, features=features, init='random', max_iter=10)
>>> cls.fit(df)
>>> df = cls.transform(df)
>>> df.head(5)
# sepal_width petal_length sepal_length petal_width class_ prediction_kmeans
0 3 4.2 5.9 1.5 1 2
1 3 4.6 6.1 1.4 1 2
2 2.9 4.6 6.6 1.3 1 2
3 3.3 5.7 6.7 2.1 2 0
4 4.2 1.4 5.5 0.2 0 1
'''
features = traitlets.List(traitlets.Unicode(),
help='List of features to cluster.')
n_clusters = traitlets.CInt(default_value=2,
help='Number of clusters to form.')
init = traitlets.Union([Matrix(), traitlets.Unicode()],
default_value='random',
help='Method for initializing the centroids.')
n_init = traitlets.CInt(default_value=1,
help='Number of centroid initializations. \
The KMeans algorithm will be run for each initialization, \
and the final results will be the best output of the n_init \
consecutive runs in terms of inertia.'
)
max_iter = traitlets.CInt(
default_value=300,
help=
'Maximum number of iterations of the KMeans algorithm for a single run.'
)
random_state = traitlets.CInt(
default_value=None,
allow_none=True,
help='Random number generation for centroid initialization. \
If an int is specified, the randomness becomes deterministic.'
)
verbose = traitlets.CBool(default_value=False,
help='If True, enable verbosity mode.')
cluster_centers = traitlets.List(traitlets.List(traitlets.CFloat()),
help='Coordinates of cluster centers.')
inertia = traitlets.CFloat(
default_value=None,
allow_none=True,
help=
'Sum of squared distances of samples to their closest cluster center.')
prediction_label = traitlets.Unicode(
default_value='prediction_kmeans',
help=
'The name of the virtual column that houses the cluster labels for each point.'
)
def __call__(self, *blocks):
return self._calculate_classes(*blocks)
def _calculate_distances_squared(self, *blocks):
N = len(blocks[0]) # they are all the same length
centroids = np.array(self.cluster_centers)
k = centroids.shape[0]
dimensions = centroids.shape[1]
distances_sq = np.zeros((k, N))
if True:
distances_square(distances_sq, centroids, *blocks)
else:
for d in range(dimensions):
for i in range(k):
distances_sq[i] += (blocks[d] - centroids[i][d])**2
return distances_sq
def _calculate_classes(self, *blocks):
distances_sq = self._calculate_distances_squared(*blocks)
classes = np.argmin(distances_sq, axis=0)
return classes
def generate_cluster_centers_random(self, dataframe, rng):
indices = rng.randint(0, len(dataframe), self.n_clusters)
return [[
dataframe.evaluate(feature, i1=i, i2=i + 1)[0]
for feature in self.features
] for i in indices]
def transform(self, dataframe):
'''
Label a DataFrame with a fitted KMeans model.
:param dataframe: A vaex DataFrame.
:returns copy: A shallow copy of the DataFrame that includes the cluster labels.
:rtype: DataFrame
'''
copy = dataframe.copy()
lazy_function = copy.add_function('kmean_predict_function',
self,
unique=True)
expression = lazy_function(*self.features)
copy.add_virtual_column(self.prediction_label,
expression,
unique=False)
return copy
def fit(self, dataframe):
'''
Fit the KMeans model to the dataframe.
:param dataframe: A vaex DataFrame.
'''
if self.init == 'random':
rng = np.random.RandomState(self.random_state)
self.run_cluster_centers = [
self.generate_cluster_centers_random(dataframe, rng)
for k in range(self.n_init)
]
else:
if self.n_init > 1:
print(
"WARNING: n_init > 1 , but init given, only doing one run")
self.run_cluster_centers = [self.init]
done = [False] * len(self.run_cluster_centers)
alldone = False
first = True
previous_inertias = None
iteration = 0
inertias_list = []
while not alldone:
new_centers, inertias = self._find_centers_and_inertias(
dataframe, done)
if self.verbose:
inertia_msges = [('--' if d else '{: 3}'.format(k))
for k, d in zip(inertias, done)]
if len(inertias) == 1:
inertia_msg = inertia_msges[0]
else:
inertia_msg = " | ".join(inertia_msges)
print('Iteration {: 4}, inertia {}'.format(
iteration, inertia_msg))
if not first: # we can only to a check after the second iteration
done = self._is_done(previous_inertias, inertias)
alldone = np.all(done)
else:
first = False
iteration += 1
if (iteration >= self.max_iter):
logger_km.debug('reached max iterations: %s', self.max_iter)
alldone = True
previous_inertias = inertias
inertias_list.append(inertias)
self.run_cluster_centers = new_centers
best_run = np.argmin(inertias)
self.inertia = inertias[best_run]
self.inertias = np.array(inertias_list)[:, best_run]
self.cluster_centers = new_centers[best_run]
def _is_done(self, inertias1, inertias2):
diffs = [(inertia1 - inertia2)
for inertia1, inertia2 in zip(inertias1, inertias2)]
return [(diff < 1e-3) for diff in diffs]
def _find_centers_and_inertias(self, dataframe, done):
done = np.array(done, dtype=np.int8)
centroids = np.array(self.run_cluster_centers)
runs = centroids.shape[0]
clusters = centroids.shape[1]
dimensions = centroids.shape[2]
# print("k =", k)
assert dimensions == len(
self.features
), "nr of dimensions for centroid should equal nr of features"
def map(*blocks): # this will be called with a chunk of the data
sumpos = np.zeros((runs, clusters, dimensions))
counts = np.zeros((runs, clusters))
inertia = np.zeros((runs))
if True:
centroid_stats(centroids, counts, sumpos, inertia, *blocks)
else:
# this is the pure python code
# although not made for multiple runs yet
distances_sq = self._calculate_distances_squared(*blocks)
classes = np.argmin(distances_sq, axis=0)
distances_sq = np.choose(classes, distances_sq)
inertia = (distances_sq).sum()
for i in range(k):
mask = classes == i
counts[i] = np.sum(mask)
for d in range(dimensions):
sumpos[i, d] += blocks[d][mask].sum()
return sumpos, counts, inertia
def reduce(x, y):
sumpos1, counts1, inertia1 = x
sumpos2, counts2, inertia2 = y
return sumpos1 + sumpos2, counts1 + counts2, inertia1 + inertia2
sumpos, counts, inertia = dataframe.map_reduce(map, reduce,
self.features)
means = (sumpos / counts[:, :, np.newaxis])
return means.tolist(), inertia
class _LinearBase(state.HasState):
features = traitlets.List(traitlets.Unicode())
binned = traitlets.CBool(True)
shape = traitlets.CInt(64)
limits = traitlets.List(traitlets.List(traitlets.CFloat(),
allow_none=True),
allow_none=True).tag(output=True)
fit_intercept = traitlets.CBool(True)
coef_ = traitlets.Union([
traitlets.List(traitlets.CFloat()),
traitlets.List(traitlets.List(traitlets.CFloat()))
]).tag(output=True)
# intercept_ = traitlets.List(traitlets.CFloat).tag(output=True)
intercept_ = traitlets.Union(
[traitlets.CFloat(),
traitlets.List(traitlets.CFloat())]).tag(output=True)
_sk_params = traitlets.Any()
prediction_name = traitlets.Unicode(default_value='linear_prediction')
def transform(self, dataset):
ds = dataset.copy()
expression = self.coef_[0] * ds[self.features[0]]
for coef, feature in zip(self.coef_, self.features[1:]):
expression = expression + coef * ds[feature]
expression = self.intercept_ + expression
ds.add_virtual_column(self.prediction_name, expression, unique=False)
return ds
def fit(self, dataset, y_expression, progress=False):
assert len(set(self.features)) == len(
self.features), "duplicate features"
if not self.binned:
X = np.array(dataset[self.features])
y = dataset.evaluate(y_expression)
m = self._make_model()
m.fit(X, y)
self.coef_ = m.coef_.tolist()
self.intercept_ = m.intercept_.tolist()
else:
limits = self.limits
if limits == []:
limits = None
binby = self.features + [y_expression]
limits, shapes = dataset.limits(binby, limits, shape=self.shape)
self.limits = listify(limits)
counts = dataset.count(binby=binby, limits=limits, shape=shapes)
mask = counts > 0
def coordinates(expression, limits, shape):
if dataset.is_category(expression):
return np.arange(dataset.category_count(expression))
else:
return dataset.bin_centers(expression, limits, shape)
centers = [
coordinates(expression, l, shape)
for expression, l, shape in zip(binby, self.limits, shapes)
]
# l = ds.bin_centers('y', limits[1], shape)
centers = np.meshgrid(*centers, indexing='ij')
centers = [c[mask] for c in centers]
# m = lin.LinearRegression(fit_intercept=self.fit_intercept)
m = self._make_model()
X = np.array(centers[:-1]).reshape(-1, len(self.features))
y = centers[-1].reshape(-1)
weights = counts[mask]
m.fit(X, y, sample_weight=weights)
self.coef_ = m.coef_.tolist()
self.intercept_ = m.intercept_.tolist()
# self._sk_params = m.get_params()
self.last_model = m
def predict(self, dataset):
X = np.array(dataset[self.features])
return self.last_model.predict(X)
class Figure(ipywebrtc.MediaStream):
"""Widget class representing a volume (rendering) using three.js"""
_view_name = Unicode('FigureView').tag(sync=True)
_view_module = Unicode('ipyvolume').tag(sync=True)
_model_name = Unicode('FigureModel').tag(sync=True)
_model_module = Unicode('ipyvolume').tag(sync=True)
_view_module_version = Unicode(semver_range_frontend).tag(sync=True)
_model_module_version = Unicode(semver_range_frontend).tag(sync=True)
volume_data = Array(default_value=None, allow_none=True).tag(sync=True, **array_cube_tile_serialization)
volume_data_original = Array(default_value=None, allow_none=True)
volume_data_max_shape = traitlets.CInt(None, allow_none=True) # TODO: allow this to be a list
eye_separation = traitlets.CFloat(6.4).tag(sync=True)
volume_data_min = traitlets.CFloat().tag(sync=True)
volume_data_max = traitlets.CFloat().tag(sync=True)
volume_show_min = traitlets.CFloat().tag(sync=True)
volume_show_max = traitlets.CFloat().tag(sync=True)
volume_clamp_min = traitlets.CBool(False).tag(sync=True)
volume_clamp_max = traitlets.CBool(False).tag(sync=True)
opacity_scale = traitlets.CFloat(1.0).tag(sync=True)
tf = traitlets.Instance(TransferFunction, allow_none=True).tag(sync=True, **ipywidgets.widget_serialization)
volume_rendering_method = traitlets.Enum(values=['NORMAL', 'MAX_INTENSITY'], default_value='NORMAL').tag(sync=True)
volume_rendering_lighting = traitlets.Bool(True).tag(sync=True)
scatters = traitlets.List(traitlets.Instance(Scatter), [], allow_none=False).tag(sync=True, **ipywidgets.widget_serialization)
meshes = traitlets.List(traitlets.Instance(Mesh), [], allow_none=False).tag(sync=True, **ipywidgets.widget_serialization)
animation = traitlets.Float(1000.0).tag(sync=True)
animation_exponent = traitlets.Float(.5).tag(sync=True)
ambient_coefficient = traitlets.Float(0.5).tag(sync=True)
diffuse_coefficient = traitlets.Float(0.8).tag(sync=True)
specular_coefficient = traitlets.Float(0.5).tag(sync=True)
specular_exponent = traitlets.Float(5).tag(sync=True)
stereo = traitlets.Bool(False).tag(sync=True)
camera_control = traitlets.Unicode(default_value='trackball').tag(sync=True)
camera_fov = traitlets.CFloat(45,min=0.1,max=179.9).tag(sync=True)
camera_center = traitlets.List(traitlets.CFloat, default_value=[0, 0, 0]).tag(sync=True)
#Tuple(traitlets.CFloat(0), traitlets.CFloat(0), traitlets.CFloat(0)).tag(sync=True)
camera = traitlets.Instance(pythreejs.Camera).tag(sync=True, **ipywidgets.widget_serialization)
@traitlets.default('camera')
def _default_camera(self):
# return pythreejs.CombinedCamera(fov=46, position=(0, 0, 2), width=400, height=500)
return pythreejs.PerspectiveCamera(fov=46, position=(0, 0, 2), width=400, height=500)
scene = traitlets.Instance(pythreejs.Scene).tag(sync=True, **ipywidgets.widget_serialization)
@traitlets.default('scene')
def _default_scene(self):
# could be removed when https://github.com/jovyan/pythreejs/issues/176 is solved
# the default for pythreejs is white, which leads the volume rendering pass to make everything white
return pythreejs.Scene(background=None)
width = traitlets.CInt(500).tag(sync=True)
height = traitlets.CInt(400).tag(sync=True)
downscale = traitlets.CInt(1).tag(sync=True)
displayscale = traitlets.CFloat(1).tag(sync=True)
capture_fps = traitlets.CFloat(None, allow_none=True).tag(sync=True)
cube_resolution = traitlets.CInt(512).tag(sync=True)
show = traitlets.Unicode("Volume").tag(sync=True) # for debugging
xlim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True)
ylim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True)
zlim = traitlets.List(traitlets.CFloat, default_value=[0, 1], minlen=2, maxlen=2).tag(sync=True)
extent = traitlets.Any().tag(sync=True)
extent_original = traitlets.Any()
matrix_projection = traitlets.List(traitlets.CFloat, default_value=[0] * 16, allow_none=True, minlen=16, maxlen=16).tag(sync=True)
matrix_world = traitlets.List(traitlets.CFloat, default_value=[0] * 16, allow_none=True, minlen=16, maxlen=16).tag(sync=True)
xlabel = traitlets.Unicode("x").tag(sync=True)
ylabel = traitlets.Unicode("y").tag(sync=True)
zlabel = traitlets.Unicode("z").tag(sync=True)
style = traitlets.Dict(default_value=ipyvolume.styles.default).tag(sync=True)
render_continuous = traitlets.Bool(False).tag(sync=True)
selector = traitlets.Unicode(default_value='lasso').tag(sync=True)
selection_mode = traitlets.Unicode(default_value='replace').tag(sync=True)
mouse_mode = traitlets.Unicode(default_value='normal').tag(sync=True)
panorama_mode = traitlets.Enum(values=['no', '360', '180'], default_value='no').tag(sync=True)
#xlim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True)
#y#lim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True)
#zlim = traitlets.Tuple(traitlets.CFloat(0), traitlets.CFloat(1)).tag(sync=True)
def __init__(self, **kwargs):
super(Figure, self).__init__(**kwargs)
self._screenshot_handlers = widgets.CallbackDispatcher()
self._selection_handlers = widgets.CallbackDispatcher()
self.on_msg(self._handle_custom_msg)
self._update_volume_data()
self.observe(self.update_volume_data, ['xlim', 'ylim', 'zlim', 'volume_data_original', 'volume_data_max_shape'])
@debounced(method=True)
def update_volume_data(self, change=None):
self._update_volume_data()
def _update_volume_data(self):
if self.volume_data_original is None:
return
if all([k <= self.volume_data_max_shape for k in self.volume_data_original.shape]):
self.volume_data = self.volume_data_original
self.extent = self.extent_original
return
shape = self.volume_data_original.shape
ex = self.extent_original
viewx, xt = grid_slice(ex[0][0], ex[0][1], shape[2], *self.xlim)
viewy, yt = grid_slice(ex[1][0], ex[1][1], shape[1], *self.ylim)
viewz, zt = grid_slice(ex[2][0], ex[2][1], shape[0], *self.zlim)
view = [slice(*viewz), slice(*viewy), slice(*viewx)]
data_view = self.volume_data_original[view]
extent = [xt, yt, zt]
data_view, extent = reduce_size(data_view, self.volume_data_max_shape, extent)
self.volume_data = np.array(data_view)
self.extent = extent
def __enter__(self):
"""Sets this figure as the current in the pylab API
Example:
>>> f1 = ipv.figure(1)
>>> f2 = ipv.figure(2)
>>> with f1:
>>> ipv.scatter(x, y, z)
>>> assert ipv.gcf() is f2
"""
import ipyvolume as ipv
self._previous_figure = ipv.gcf()
ipv.figure(self)
def __exit__(self, type, value, traceback):
import ipyvolume as ipv
ipv.figure(self._previous_figure)
del self._previous_figure
def screenshot(self, width=None, height=None, mime_type='image/png'):
self.send({'msg':'screenshot', 'width':width, 'height':height, 'mime_type':mime_type})
def on_screenshot(self, callback, remove=False):
self._screenshot_handlers.register_callback(callback, remove=remove)
def _handle_custom_msg(self, content, buffers):
if content.get('event', '') == 'screenshot':
self._screenshot_handlers(content['data'])
elif content.get('event', '') == 'selection':
self._selection_handlers(content['data'])
def on_selection(self, callback, remove=False):
self._selection_handlers.register_callback(callback, remove=remove)
def project(self, x, y, z):
W = np.matrix(self.matrix_world).reshape((4,4)) .T
P = np.matrix(self.matrix_projection).reshape((4,4)).T
M = np.dot(P, W)
x = np.asarray(x)
vertices = np.array([x, y, z, np.ones(x.shape)])
screen_h = np.tensordot(M, vertices, axes=(1, 0))
xy = screen_h[:2] / screen_h[3]
return xy
class QMTSlopeTile(StrictHasTraits, object):
name = "qmtslope"
urltemplate = "https://assets.agi.com/stk-terrain/world/{z}/{x}/{y}.terrain?v=1.31376.0"
geodetic = quantized_mesh_tile.global_geodetic.GlobalGeodetic(True)
opacity = traitlets.CFloat(max=1.0, min=0.0, default_value=1.0)
line = traitlets.CBool(default_value=False)
@property
def storage_key(self):
key = self.name
if self.line:
key = key + "_line_True"
return key
@classmethod
def spanning_tile_coords(cls, xyzp):
mb = mercantile.bounds(xyzp)
z = xyzp.z - 1
if mb.north >= 49.0 and z > 14:
z = 14
elif z > 15:
z = 15
spanning_tiles = set(
cls.geodetic.LonLatToTile(lat, lon, z)
for lat, lon in itertools.product((mb.west, mb.east), (mb.north,
mb.south)))
return [(x, y, z) for x, y in spanning_tiles]
@classmethod
def load_qmt(cls, x, y, z, content):
cls.geodetic.TileBounds(x, y, z)
bounds = dict(
zip(("west", "south", "east", "north"),
cls.geodetic.TileBounds(x, y, z)))
tile = quantized_mesh_tile.TerrainTile(**bounds)
tile.fromStringIO(io.BytesIO(content))
return tile
@classmethod
def vnorm(self, vectors):
#No axis arg in numpy 1.6 norm
return sum([vectors[..., i]**2 for i in (0, 1, 2)])**.5
@classmethod
def triangle_slope_angles(cls, qt_coords, qt_triangles):
qt_meters = qt_coords.copy()
qt_meters[...,
0], qt_meters[...,
1] = mercantile.xy(qt_coords[..., 0],
qt_coords[..., 1])
tri_crosses = numpy.cross(
qt_meters[qt_triangles[..., 0]] - qt_meters[qt_triangles[..., 1]],
qt_meters[qt_triangles[..., 0]] - qt_meters[qt_triangles[..., 2]])
off_z_angle = numpy.rad2deg(
numpy.arccos(tri_crosses[:, 2] / cls.vnorm(tri_crosses)))
return off_z_angle
@ndb.tasklet
def fetch_tile_data(self, x, y, z):
context = ndb.get_context()
tile_data = yield context.urlfetch(
self.urltemplate.format(x=x, y=y, z=z))
if tile_data.status_code != 200:
logging.error("error fetching qmt: %r url: %s result: %s",
(x, y, z), self.urltemplate.format(x=x, y=y,
z=z), result)
raise ValueError("error fetching qmt: %r url: %s result: %s",
(x, y, z), self.urltemplate.format(x=x, y=y,
z=z), result)
logging.info("loading qmt file: %s", ((x, y, z), ))
qmt = self.load_qmt(x, y, z, tile_data.content)
logging.info("generating mesh array: %s", ((x, y, z), ))
raise ndb.Return({
"coords": numpy.array(qmt.getVerticesCoordinates()),
"triangles": numpy.array(numpy.array(qmt.indices))
})
@ndb.tasklet
def render_async(self, tile):
tile = mercantile.Tile(**tile)
cache = memcache._CLIENT
targets = self.spanning_tile_coords(tile)
tkeys = {str(t): t for t in targets}
cached_coords = yield cache.get_multi_async(
tkeys, key_prefix="%s+tilearray+" % self.name)
newkeys = set(tkeys) - set(cached_coords)
newdata = yield tuple(self.fetch_tile_data(*tkeys[c]) for c in newkeys)
new = dict(zip(newkeys, newdata))
yield cache.set_multi_async(new,
key_prefix="%s+tilearray+" % self.name)
qm_coords = []
qm_triangles = []
ind = 0
for qmt in list(cached_coords.values()) + list(new.values()):
qm_coords.append(qmt["coords"])
qm_triangles.append(qmt["triangles"] + ind)
ind += len(qm_coords[-1])
qm_coords = numpy.concatenate(qm_coords, axis=0)
qm_triangles = numpy.concatenate(qm_triangles, axis=0).reshape((-1, 3))
raise ndb.Return(self._render_from_qmt(tile, qm_coords, qm_triangles))
def render(self, tile):
tile = mercantile.Tile(**tile)
qmt_data = {(x, y, z):
requests.get(self.urltemplate.format(x=x, y=y,
z=z)).content
for x, y, z in self.spanning_tile_coords(tile)}
logging.info("loading qmt files: %s", qmt_data.keys())
qmts = [self.load_qmt(x, y, z, d) for (x, y, z), d in qmt_data.items()]
logging.info("generating mesh arrays")
qm_coords = []
qm_triangles = []
ind = 0
for qmt in qmts:
qm_coords.append(numpy.array(qmt.getVerticesCoordinates()))
qm_triangles.append(numpy.array(qmt.indices) + ind)
ind += len(qm_coords[-1])
qm_coords = numpy.concatenate(qm_coords, axis=0)
qm_triangles = numpy.concatenate(qm_triangles, axis=0).reshape((-1, 3))
return self._render_from_qmt(tile, qm_coords, qm_triangles)
def _render_from_qmt(self, tile, qm_coords, qm_triangles):
logging.info("calculating slope angles: %i", len(qm_triangles))
slope_angles = self.triangle_slope_angles(qm_coords, qm_triangles)
logging.info("coloring slope")
mb = mercantile.bounds(tile)
qm_xs = (qm_coords[..., 0] - mb.west) / ((mb.east - mb.west) / 255)
qm_ys = (qm_coords[..., 1] - mb.north) / ((mb.south - mb.north) / 255)
qm_colors = angle_to_rbga(slope_angles)
qm_pix = numpy.empty((len(qm_coords), 2), dtype=int)
qm_pix[..., 0] = qm_xs + 128
qm_pix[..., 1] = qm_ys + 128
logging.info("rendering tile")
rbuff = Image.new("RGBA", (256 * 2, 256 * 2))
rdraw = ImageDraw.ImageDraw(rbuff)
for ti in range(len(qm_triangles)):
rdraw.polygon(map(tuple, qm_pix[qm_triangles[ti]]),
fill=tuple(qm_colors[ti]),
outline=(0, 0, 0, 255) if self.line else None)
res = rbuff.crop((128, 128, 255 + 128, 255 + 128))
return res
class PCA(Transformer):
'''Transform a set of features using a Principal Component Analysis.
Example:
>>> import vaex
>>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10])
>>> df
# x y
0 2 -2
1 5 3
2 7 0
3 2 0
4 15 10
>>> pca = vaex.ml.PCA(n_components=2, features=['x', 'y'])
>>> pca.fit_transform(df)
# x y PCA_0 PCA_1
0 2 -2 5.92532 0.413011
1 5 3 0.380494 -1.39112
2 7 0 0.840049 2.18502
3 2 0 4.61287 -1.09612
4 15 10 -11.7587 -0.110794
'''
# title = traitlets.Unicode(default_value='PCA', read_only=True).tag(ui='HTML')
n_components = traitlets.Int(
help=
'Number of components to retain. If None, all the components will be retained.'
).tag(ui='IntText')
prefix = traitlets.Unicode(default_value="PCA_", help=help_prefix)
progress = traitlets.CBool(
default_value=False,
help='If True, display a progressbar of the PCA fitting process.').tag(
ui='Checkbox')
eigen_vectors_ = traitlets.List(
traitlets.List(traitlets.CFloat()),
help='The eigen vectors corresponding to each feature').tag(
output=True)
eigen_values_ = traitlets.List(
traitlets.CFloat(),
help='The eigen values that correspond to each feature.').tag(
output=True)
means_ = traitlets.List(traitlets.CFloat(),
help='The mean of each feature').tag(output=True)
@traitlets.default('n_components')
def get_n_components_default(self):
return len(self.features)
def fit(self, df):
'''Fit the PCA model to the DataFrame.
:param df: A vaex DataFrame.
'''
self.n_components = self.n_components or len(self.features)
assert self.n_components >= 2, 'At least two features are required.'
assert self.n_components <= len(
self.features), 'Can not have more components than features.'
C = df.cov(self.features, progress=self.progress)
eigen_values, eigen_vectors = np.linalg.eigh(C)
indices = np.argsort(eigen_values)[::-1]
self.means_ = df.mean(self.features, progress=self.progress).tolist()
self.eigen_vectors_ = eigen_vectors[:, indices].tolist()
self.eigen_values_ = eigen_values[indices].tolist()
def transform(self, df, n_components=None):
'''Apply the PCA transformation to the DataFrame.
:param df: A vaex DataFrame.
:param n_components: The number of PCA components to retain.
:return copy: A shallow copy of the DataFrame that includes the PCA components.
:rtype: DataFrame
'''
n_components = n_components or self.n_components
copy = df.copy()
name_prefix_offset = 0
eigen_vectors = np.array(self.eigen_vectors_)
while self.prefix + str(name_prefix_offset) in copy.get_column_names(
virtual=True, strings=True):
name_prefix_offset += 1
expressions = [
copy[feature] - mean
for feature, mean in zip(self.features, self.means_)
]
for i in range(n_components):
v = eigen_vectors[:, i]
expr = dot_product(expressions, v)
name = self.prefix + str(i + name_prefix_offset)
copy[name] = expr
return copy
class RobustScaler(Transformer):
''' The RobustScaler removes the median and scales the data according to a
given percentile range. By default, the scaling is done between the 25th and
the 75th percentile. Centering and scaling happens independently for each
feature (column).
Example:
>>> import vaex
>>> df = vaex.from_arrays(x=[2,5,7,2,15], y=[-2,3,0,0,10])
>>> df
# x y
0 2 -2
1 5 3
2 7 0
3 2 0
4 15 10
>>> scaler = vaex.ml.MaxAbsScaler(features=['x', 'y'])
>>> scaler.fit_transform(df)
# x y robust_scaled_x robust_scaled_y
0 2 -2 -0.333686 -0.266302
1 5 3 -0.000596934 0.399453
2 7 0 0.221462 0
3 2 0 -0.333686 0
4 15 10 1.1097 1.33151
'''
with_centering = traitlets.CBool(
default_value=True,
help='If True, remove the median.').tag(ui='Checkbox')
with_scaling = traitlets.CBool(
default_value=True,
help=
'If True, scale each feature between the specified percentile range.'
).tag(ui='Checkbox')
percentile_range = traitlets.Tuple(
default_value=(25, 75),
help='The percentile range to which to scale each feature to.').tag(
).tag(ui='FloatRangeSlider')
prefix = traitlets.Unicode(default_value="robust_scaled_",
help=help_prefix).tag(ui='Text')
center_ = traitlets.List(
traitlets.CFloat(),
default_value=None,
help='The median of each feature.').tag(output=True)
scale_ = traitlets.List(
traitlets.CFloat(),
default_value=None,
help='The percentile range for each feature.').tag(output=True)
def fit(self, df):
'''
Fit RobustScaler to the DataFrame.
:param df: A vaex DataFrame.
'''
# check the quantile range
q_min, q_max = self.percentile_range
if not 0 <= q_min <= q_max <= 100:
raise ValueError('Invalid percentile range: %s' %
(str(self.percentile_range)))
if self.with_centering:
self.center_ = df.percentile_approx(expression=self.features,
percentage=50).tolist()
if self.with_scaling:
self.scale_ = (df.percentile_approx(expression=self.features,
percentage=q_max) -
df.percentile_approx(expression=self.features,
percentage=q_min)).tolist()
def transform(self, df):
'''
Transform a DataFrame with a fitted RobustScaler.
:param df: A vaex DataFrame.
:returns copy: a shallow copy of the DataFrame that includes the scaled features.
:rtype: DataFrame
'''
copy = df.copy()
for i, feature in enumerate(self.features):
name = self.prefix + feature
expr = copy[feature]
if self.with_centering:
expr -= self.center_[i]
if self.with_scaling:
expr /= self.scale_[i]
copy[name] = expr
return copy
class Generator(t.HasTraits):
'''Generator Model'''
name = t.CUnicode(default_value='GenCo0', help='Name of Generator (str)')
generator_bus = t.CUnicode(default_value='Bus0',
help='Bus of Generator (str)')
generator_voltage = t.CFloat(
default_value=1.0, help='Nominal voltage of the generator (p.u.)')
base_power = t.CFloat(default_value=100.0,
help='Base power of the generator (MVA)')
generation_type = t.Enum(['COAL', 'NATURALGAS', 'WIND'],
default_value='COAL')
minimum_up_time = t.CInt(default_value=0,
min=0,
help='Minimum up time (hrs)')
minimum_down_time = t.CInt(default_value=0,
min=0,
help='Minimum down time (hrs)')
ramp_up_rate = t.CFloat(default_value=0,
min=0,
help='Ramp up rate (MW/hr)')
ramp_down_rate = t.CFloat(default_value=0,
min=0,
help='Ramp down rate (MW/hr)')
maximum_real_power = t.CFloat(default_value=0,
min=0,
help='Capacity of Generator (MW)')
minimum_real_power = t.CFloat(default_value=0,
min=0,
help='Minimum generation (MW)')
maximum_imag_power = t.CFloat(default_value=0,
help='Maximum reactive generation (MVAR)')
minimum_imag_power = t.CFloat(default_value=0,
help='Minimum reactive generation (MVAR)')
initial_real_power = t.CFloat(default_value=0,
min=0,
help='Initial power generation (MW)')
initial_imag_power = t.CFloat(default_value=0,
min=0,
help='Initial power generation (MVAR)')
initial_status = t.CBool(default_value=True,
min=0,
help='Initial status (bool)')
startup_time = t.CInt(default_value=0, min=0, help='Startup time (hrs)')
shutdown_time = t.CInt(default_value=0, min=0, help='Shutdown time (hrs)')
nsegments = t.CInt(default_value=2,
min=MINIMUM_COST_CURVE_SEGMENTS,
max=MAXIMUM_COST_CURVE_SEGMENTS,
help='Number of data points for piecewise linear')
cost_curve_points = tt.Array(default_value=[0, 0, 0],
minlen=(MINIMUM_COST_CURVE_SEGMENTS + 1),
maxlen=(MAXIMUM_COST_CURVE_SEGMENTS + 1))
cost_curve_values = tt.Array(default_value=[0, 0, 0],
minlen=(MINIMUM_COST_CURVE_SEGMENTS + 1),
maxlen=(MAXIMUM_COST_CURVE_SEGMENTS + 1))
noload_cost = t.CFloat(default_value=0,
min=0,
help='No-Load Cost of a Generator ($/hr)')
startup_cost = t.CFloat(default_value=0,
min=0,
help='Startup Cost of a Generator ($/hr)')
inertia = t.CFloat(allow_none=True,
default_value=None,
min=0,
help='Inertia of generator (NotImplemented)')
droop = t.CFloat(allow_none=True,
default_value=None,
min=0,
help='Droop of generator (NotImplemented)')
def __init__(self, *args, **kwargs):
super(Generator, self).__init__(*args, **kwargs)
@property
def _npoints(self):
return self.nsegments + 1
@property
def ramp_rate(self):
raise AttributeError(
"'{class_name}' object has no attribute 'ramp_rate'. Try 'ramp_up_rate' or 'ramp_down_rate'."
.format(class_name=self.__class__.__name__))
@ramp_rate.setter
def ramp_rate(self, v):
self.ramp_up_rate = v
self.ramp_down_rate = v
@t.observe('noload_cost')
def _callback_noload_cost_update_points_values(self, change):
self.cost_curve_values = [change['new']] * self._npoints
return change['new']
@t.observe('minimum_real_power')
def _callback_minimum_real_power_update_points_values(self, change):
self.cost_curve_points = np.linspace(change['new'],
self.maximum_real_power,
self._npoints)
return change['new']
@t.observe('maximum_real_power')
def _callback_maximum_real_power_update_points_values(self, change):
self.cost_curve_points = np.linspace(self.minimum_real_power,
change['new'], self._npoints)
self.ramp_rate = self.maximum_real_power
return change['new']
@t.observe('nsegments')
def _callback_nsegments_update_points_values(self, change):
self.cost_curve_points = np.linspace(self.minimum_real_power,
self.maximum_real_power,
change['new'] + 1)
self.cost_curve_values = [self.noload_cost] * (change['new'] + 1)
return change['new']
@t.validate('cost_curve_points', 'cost_curve_values')
def _validate_max_length(self, proposal):
if not len(proposal['value']) == self._npoints:
raise t.TraitError(
'len({class_name}().{trait_name}) must be equal to {class_name}().nsegments + 1. Proposed {trait_name} is {proposal}'
.format(class_name=proposal['owner'].__class__.__name__,
trait_name=proposal['trait'].name,
proposal=proposal['value']))
return proposal['value']
@t.validate('ramp_up_rate', 'ramp_down_rate', 'initial_real_power',
'initial_imag_power')
def _less_than_maximum_real_power_check(self, proposal):
if not proposal['value'] <= self.maximum_real_power:
raise t.TraitError(
'{class_name}().{trait_name} must be a less than or equal to {class_name}().maximum_real_power.'
.format(class_name=proposal['owner'].__class__.__name__,
trait_name=proposal['trait'].name))
else:
return proposal['value']
