class FlagSetter(Stream):
"""Stream for setting flags
Most useful in context of a parambokeh widget, e.g.:
from explorer.plots import FlagSetter
import parambokeh
flag_setter = FlagSetter(filter_stream=filter_stream, flags=data.flags, bad_flags=data.flags)
parambokeh.Widgets(flag_setter, callback=flag_setter.event, push=False, on_init=True)
Where `filter_stream` has been previously defined and connected to other plots
for which you want to see points with certain flags shown/hidden/etc.
"""
flags = param.ListSelector(
default=[],
objects=[],
doc="""
Flags to select""",
)
bad_flags = param.ListSelector(
default=[],
doc="""
Flags to ignore""",
)
def __init__(self, filter_stream, **kwargs):
super(FlagSetter, self).__init__(**kwargs)
self.filter_stream = filter_stream
def event(self, **kwargs):
self.filter_stream.event(**kwargs)
class P(param.Parameterized):
e = param.ListSelector(default=[5], objects=[5, 6, 7])
f = param.ListSelector(default=10)
h = param.ListSelector(default=None)
g = param.ListSelector(default=None, objects=[7, 8])
i = param.ListSelector(default=[7],
objects=[9],
check_on_set=False)
class StockScreener(param.Parameterized):
df = param.DataFrame(precedence=-1)
index = param.ListSelector()
normalize = param.Boolean(default=True)
start = param.Date()
def __init__(self, df, **params):
start = dt.date(year=df.index[0].year,
month=df.index[0].month,
day=df.index[0].day)
end = dt.date(year=df.index[-1].year,
month=df.index[-1].month,
day=df.index[-1].day)
super(StockScreener, self).__init__(df=df, start=start, **params)
self.param.start.bounds = (start, end)
columns = list(self.df.columns)
self.param.index.objects = columns
self.index = columns[:5]
@param.depends('index', 'normalize', 'start')
def update_plot(self):
pos = self.df.index.get_loc(self.start, method='bfill')
dfp = self.df.iloc[pos:][self.index]
if self.normalize:
dfp = 100 * dfp / dfp.iloc[0]
return dfp.hvplot(group_label='Ticker')
def panel(self):
return pn.Row(
pn.panel(self.param, widgets={'start': pn.widgets.DateSlider}),
self.update_plot)
class Derived(base):
only = param.ListSelector(
default,
objects=list(tunable),
doc="When performing hyperparameter optimization, only optimize "
"these parameters",
)
class MetricCheckboxGroup(param.Parameterized):
metrics = param.ListSelector(default=[])
def __init__(self, available_metrics, **kwargs):
self.param.metrics.objects = sorted(available_metrics)
super().__init__(**kwargs)
class TestSet(param.Parameterized):
numpy_params = ['r']
pandas_params = ['s','t','u']
conditionally_unsafe = ['f', 'o']
a = param.Integer(default=5, doc='Example doc', bounds=(2,30), inclusive_bounds=(True, False))
b = param.Number(default=4.3, allow_None=True)
c = param.String(default='foo')
d = param.Boolean(default=False)
e = param.List([1,2,3], class_=int)
f = param.List([1,2,3])
g = param.Date(default=datetime.datetime.now())
h = param.Tuple(default=(1,2,3), length=3)
i = param.NumericTuple(default=(1,2,3,4))
j = param.XYCoordinates(default=(32.1, 51.5))
k = param.Integer(default=1)
l = param.Range(default=(1.1,2.3), bounds=(1,3))
m = param.String(default='baz', allow_None=True)
n = param.ObjectSelector(default=3, objects=[3,'foo'], allow_None=False)
o = param.ObjectSelector(default=simple_list, objects=[simple_list], allow_None=False)
p = param.ListSelector(default=[1,4,5], objects=[1,2,3,4,5,6])
q = param.CalendarDate(default=datetime.date.today())
r = None if np is None else param.Array(default=ndarray)
s = None if pd is None else param.DataFrame(default=df1, columns=2)
t = None if pd is None else param.DataFrame(default=pd.DataFrame(
{'A':[1,2,3], 'B':[1.1,2.2,3.3]}), columns=(1,4), rows=(2,5))
u = None if pd is None else param.DataFrame(default=df2, columns=['A', 'B'])
v = param.Dict({'1':2})
class NetPresentValue(pm.Parameterized):
cashflows = pm.ListSelector(default=[], objects=[])
def __init__(self, discount_rate: InterestRate, cash_flows: list,
**params):
self.discount_rate = discount_rate
self.cash_flows = cash_flows
super(NetPresentValue, self).__init__(**params)
class Reference(param.Parameterized):
point_ref = param.ListSelector(
objects=["Pays", "Région", "Département", "Intercommune", "Commune"],
label="Point de référence",
)
donnees_infra = param.Action(
lambda x: x, doc="""Données Infra-Communales""", precedence=0.7
)
class RangeToolLink(Link):
"""
The RangeToolLink sets up a link between a RangeTool on the source
plot and the axes on the target plot. It is useful for exploring
a subset of a larger dataset in more detail. By default it will
link along the x-axis but using the axes parameter both axes may
be linked to the tool.
"""
axes = param.ListSelector(default=['x'],
objects=['x', 'y'],
doc="""
Which axes to link the tool to.""")
class _BigDumbParams(param.Parameterized):
action = param.Action(default_action, allow_None=True)
array = param.Array(np.array([1.0, 2.0]))
boolean = param.Boolean(True, allow_None=True)
callable = param.Callable(default_action, allow_None=True)
class_selector = param.ClassSelector(int, is_instance=False, allow_None=True)
color = param.Color("#FFFFFF", allow_None=True)
composite = param.Composite(["action", "array"], allow_None=True)
try:
data_frame = param.DataFrame(
pd.DataFrame({"A": 1.0, "B": np.arange(5)}), allow_None=True
)
except TypeError:
data_frame = param.DataFrame(pd.DataFrame({"A": 1.0, "B": np.arange(5)}))
date = param.Date(datetime.now(), allow_None=True)
date_range = param.DateRange((datetime.min, datetime.max), allow_None=True)
dict_ = param.Dict({"foo": "bar"}, allow_None=True, doc="dict means dictionary")
dynamic = param.Dynamic(default=default_action, allow_None=True)
file_selector = param.FileSelector(
os.path.join(FILE_DIR_DIR, "LICENSE"),
path=os.path.join(FILE_DIR_DIR, "*"),
allow_None=True,
)
filename = param.Filename(
os.path.join(FILE_DIR_DIR, "LICENSE"), allow_None=True
)
foldername = param.Foldername(os.path.join(FILE_DIR_DIR), allow_None=True)
hook_list = param.HookList(
[CallableObject(), CallableObject()], class_=CallableObject, allow_None=True
)
integer = param.Integer(10, allow_None=True)
list_ = param.List([1, 2, 3], allow_None=True, class_=int)
list_selector = param.ListSelector([2, 2], objects=[1, 2, 3], allow_None=True)
magnitude = param.Magnitude(0.5, allow_None=True)
multi_file_selector = param.MultiFileSelector(
[],
path=os.path.join(FILE_DIR_DIR, "*"),
allow_None=True,
check_on_set=True,
)
number = param.Number(-10.0, allow_None=True, doc="here is a number")
numeric_tuple = param.NumericTuple((5.0, 10.0), allow_None=True)
object_selector = param.ObjectSelector(
False, objects={"False": False, "True": 1}, allow_None=True
)
path = param.Path(os.path.join(FILE_DIR_DIR, "LICENSE"), allow_None=True)
range_ = param.Range((-1.0, 2.0), allow_None=True)
series = param.Series(pd.Series(range(5)), allow_None=True)
string = param.String("foo", allow_None=True, doc="this is a string")
tuple_ = param.Tuple((3, 4, "fi"), allow_None=True)
x_y_coordinates = param.XYCoordinates((1.0, 2.0), allow_None=True)
class Example(BaseClass):
"""An example Parameterized class"""
timestamps = []
boolean = param.Boolean(True, doc="A sample Boolean parameter")
color = param.Color(default="#FFFFFF")
date = param.Date(dt.date(2017, 1, 1), bounds=wired.DATE_BOUNDS)
dataframe = param.DataFrame(pd.util.testing.makeDataFrame().iloc[:3])
select_string = param.ObjectSelector(default="yellow", objects=["red", "yellow", "green"])
select_fn = param.ObjectSelector(default=list, objects=[list, set, dict])
int_list = param.ListSelector(default=[3, 5], objects=[1, 3, 5, 7, 9], precedence=0.5)
single_file = param.FileSelector(path="../../*/*.py*", precedence=0.5)
multiple_files = param.MultiFileSelector(path="../../*/*.py?", precedence=0.5)
record_timestamp = param.Action(
lambda x: x.timestamps.append(dt.datetime.utcnow()),
doc="""Record timestamp.""",
precedence=0.7,
)
class MyParameterized(param.Parameterized):
enable = param.Boolean(True,
doc="A sample Boolean parameter",
allow_None=True)
what_proportion = param.Magnitude(default=0.9)
age = param.Number(49,
bounds=(0, 100),
doc="Any Number between 0 to 100")
how_many = param.Integer()
favorite_quote = param.String(default="Hello, world!")
choose_file_or_folder = param.Path(search_paths='./')
choose_folder = param.Foldername(search_paths="./")
choose_file = param.Filename(search_paths="./")
select_a_file = param.FileSelector(path='./*')
select_multiple_files = param.MultiFileSelector(path='./*')
favorite_color = param.ObjectSelector(
default="green", objects=["red", "yellow", "green"])
favorite_fruit = param.Selector(default="Apple",
objects=["Orange", "Apple", "Mango"])
select_multiple = param.ListSelector(default=[3, 5],
objects=[1, 2, 3, 4, 5])
birthday = param.CalendarDate(dt.date(2017, 1, 1),
bounds=(dt.date(2017, 1,
1), dt.date(2017, 2, 1)))
appointment = param.Date(dt.datetime(2017, 1, 1),
bounds=(dt.datetime(2017, 1, 1),
dt.datetime(2017, 2, 1)))
least_favorite_color = param.Color(default='#FF0000')
dataset = param.DataFrame(pd.util.testing.makeDataFrame().iloc[:3])
this_strange_thing = param.Tuple(default=(False, ), allow_None=True)
some_numbers = param.NumericTuple(default=(1, 2, 3.0, 4.0))
home_city = param.XYCoordinates(default=(-111.65, 40.23))
bounds = param.Range(default=(-10, 10))
class Facet(param.Parameterized):
by = param.List(default=[],
class_=FacetFilter,
doc="""
Fields to facet by.""")
layout = param.ClassSelector(default=None,
class_=(str, dict),
doc="""
How to lay out the facets.""")
sort = param.ListSelector(default=[],
objects=[],
doc="""
List of fields to sort by.""")
reverse = param.Boolean(default=False,
doc="""
Whether to reverse the sort order.""")
def __init__(self, **params):
super().__init__(**params)
self._sort_widget = pn.widgets.MultiSelect(
options=self.param.sort.objects,
sizing_mode='stretch_width',
size=len(self.sort),
value=self.sort,
)
self._sort_widget.link(self, value='sort')
self._reverse_widget = pn.widgets.Checkbox(value=self.reverse,
name='Reverse',
margin=(5, 0, 0, 10))
self._reverse_widget.link(self, value='reverse')
@param.depends('sort:objects', watch=True)
def _update_options(self):
self._sort_widget.options = self.param.sort.objects
@classmethod
def from_spec(cls, spec, schema):
"""
Creates a Facet object from a schema and a set of fields.
"""
by = []
for by_spec in spec.pop('by', []):
if isinstance(by_spec, str):
f = Filter.from_spec({
'type': 'facet',
'field': by_spec
}, schema)
elif isinstance(by_spec, dict):
f = Filter.from_spec(dict(by_spec, type='facet'), schema)
elif isinstance(by_spec, Filter):
f = by_spec
by.append(f)
sort = spec.pop('sort', [b.field for b in by])
sorter = cls(by=by, **spec)
sorter.param.sort.objects = sort
return sorter
def get_sort_key(self, views):
sort_key = []
for field in self.sort:
values = [v.get_value(field) for v in views]
if values:
sort_key.append(values[0])
return tuple(sort_key)
@property
def filters(self):
return product(*[filt.filters for filt in self.by])
class BaseMultiplePage(Page):
"""A view for multiple Ticker requests
The user can select all or multiple endpoints and the help text, code and result will be
presented."""
all_endpoints = param.Boolean(
default=False,
doc="If True all endpoints should be requested otherwise only multiple",
)
endpoints = param.ListSelector(
default=["assetProfile"],
objects=sorted(Ticker._ENDPOINTS), # pylint: disable=protected-access
)
@param.depends("all_endpoints")
def _endpoints_widget(
self,
):
if not self.all_endpoints:
return pn.Param(
self.param.endpoints,
widgets={
"endpoints": {"height": 600},
},
)
return None
@param.depends(
"all_endpoints",
"endpoints",
)
def _help(
self,
):
if self.all_endpoints:
return pnx_help(Ticker.all_endpoints)
return pnx_help(Ticker.get_endpoints)
@param.depends(
"symbols",
"all_endpoints",
"endpoints",
)
def _code(
self,
):
if self.all_endpoints:
code = f"Ticker('{self.symbols}').all_endpoint"
else:
code = f"Ticker('{self.symbols}').get_endpoints({self.endpoints})"
return code_card(code=code)
@param.depends(
"symbols",
"all_endpoints",
"endpoints",
)
@PROGRESS.report(message="Requesting Multiple Endpoints from Yahoo Finance")
def _data(
self,
):
if self.all_endpoints:
data = YahooQueryService.get_data(
self.symbols,
"all_endpoints",
)
else:
if not self.endpoints:
return pnx.InfoAlert("Please select one or more Endpoints")
data = YahooQueryService.get_data(
self.symbols,
"get_endpoints",
)(self.endpoints)
return pnx_json(data)
def _selections(
self,
):
return pn.Column(
pnx.SubHeader(" Selections"),
self.param.all_endpoints,
self._endpoints_widget,
sizing_mode="fixed",
)
def view(
self,
) -> pn.viewable.Viewable:
"""The main view of the BasePage
Returns:
pn.viewable.Viewable: The main view of the BasePage
"""
return pn.Column(
self._selections,
self._data,
self._code,
self._help,
sizing_mode="stretch_width",
)
class HpcConnect(param.Parameterized):
uit_client = param.ClassSelector(Client)
system = param.ObjectSelector(default=HPC_SYSTEMS[0], objects=HPC_SYSTEMS)
login_node = param.ObjectSelector(default=None,
objects=[None],
label='Login Node')
exclude_nodes = param.ListSelector(default=list(),
objects=[],
label='Exclude Nodes')
connected = param.Boolean(default=False, allow_None=True)
connect_btn = param.Action(lambda self: self.connect(), label='Connect')
disconnect_btn = param.Action(lambda self: self.disconnect(),
label='Disconnect')
connection_status = param.String(default='Not Connected', label='Status')
ready = param.Boolean(default=False, precedence=-1)
_next_stage = param.Selector()
next_stage = param.Selector()
def __init__(self, **params):
super().__init__(**params)
self.advanced_pn = None
self.system_pn = pn.Column(
pn.panel(self, parameters=['system'], show_name=False),
pn.Spacer(),
name='HPC System',
)
@param.depends('system', 'uit_client', watch=True)
def update_node_options(self):
if self.uit_client is not None:
options = self.uit_client.login_nodes[self.system]
self.param.exclude_nodes.objects = options
options = options.copy()
options.insert(0, None)
self.param.login_node.objects = options
self.param.login_node.names = {'Random': None}
@param.depends('login_node', watch=True)
def update_exclude_nodes_visibility(self):
self.param.exclude_nodes.precedence = 1 if self.login_node is None else -1
if self.login_node is None:
self.advanced_pn.extend([
self.param.exclude_nodes.label,
pn.Param(
self.param.exclude_nodes,
widgets={'exclude_nodes': pn.widgets.CrossSelector},
),
])
else:
self.advanced_pn[:] = self.advanced_pn[:1]
@param.depends('_next_stage', watch=True)
def update_next_stage(self):
self.next_stage = self._next_stage
def connect(self):
system = None if self.login_node is not None else self.system
try:
self.connected = None
retry = self.login_node is None
self.connection_status = self.uit_client.connect(
system=system,
login_node=self.login_node,
exclude_login_nodes=self.exclude_nodes,
retry_on_failure=retry,
)
except UITError as e:
log.exception(e)
self.exclude_nodes.append(self.uit_client.login_node)
self.disconnect()
self.system_pn[-1] = pn.pane.Alert(
f'{e}<br/>Try connecting again.'.format(alert_type='danger'),
alert_type='danger',
width=500)
else:
self.connected = self.uit_client.connected
self.ready = self.connected
def disconnect(self):
self.param.connect_btn.label = 'Connect'
self.connection_status = 'Not Connected'
self.system_pn[-1] = pn.Spacer()
self.login_node = None
self.connected = False
@param.depends('connected')
def view(self):
header = '# Connect to HPC System'
spn = pn.widgets.indicators.LoadingSpinner(value=True,
color='primary',
aspect_ratio=1,
width=0)
connect_btn = pn.Param(
self.param.connect_btn,
widgets={
'connect_btn': {
'button_type': 'success',
'width': 100,
}
},
)[0]
connect_btn.js_on_click(args={
'btn': connect_btn,
'spn': spn
},
code='btn.visible=false; spn.width=50;')
if self.connected is None:
content = spn
elif self.connected is False:
self.advanced_pn = pn.panel(
self,
parameters=['login_node', 'exclude_nodes'],
widgets={'exclude_nodes': pn.widgets.CrossSelector},
show_name=False,
name='Advanced Options',
)
if self.login_node is None:
self.advanced_pn.insert(1, self.param.exclude_nodes.label)
content = pn.Column(
pn.layout.Tabs(self.system_pn, self.advanced_pn), connect_btn,
spn)
else:
self.param.connect_btn.label = 'Re-Connect'
btns = pn.Param(
self,
parameters=['connect_btn', 'disconnect_btn'],
widgets={
'disconnect_btn': {
'button_type': 'danger',
'width': 100
},
'connect_btn': {
'button_type': 'success',
'width': 100
}
},
show_name=False,
default_layout=pn.Row,
)
return pn.Column(
header,
btns,
pn.panel(self,
parameters=['connection_status'],
show_name=False,
width=400),
)
return pn.Column(header, content, width=500)
def panel(self):
return pn.panel(self.view)
class UMAPPlotClass(param.Parameterized):
lista_opciones = ['leiden'] + ['partido_1', 'partido_2', 'partido_3'] + ['provincia', 'autonomia'] + \
[i for i in df.keys() if (i.startswith('porcentaje_')) &
(i not in ['porcentaje_1', 'porcentaje_2', 'porcentaje_3'])]
lista_autonomias = sorted(list(dict.fromkeys(df['autonomia'])))
obj_opciones = param.ObjectSelector(default='leiden',
objects=lista_opciones)
obj_autonomias = param.ListSelector(default=lista_autonomias,
objects=lista_autonomias)
def color_mapper(self, c2, c1, mix):
if np.isnan(mix): mix = 0
c1 = np.array(mpl.colors.to_rgb(c1))
c2 = np.array(mpl.colors.to_rgb(c2))
return mpl.colors.to_hex((1 - mix) * c1 + mix * c2)
def crear_fig(
): # Esto técnicamente está mal pero funciona, así que ni lo toco
fig = figure(
plot_height=700,
plot_width=700,
tools='box_zoom,reset,pan,wheel_zoom,lasso_select,undo,redo',
sizing_mode='scale_width',
output_backend="webgl",
toolbar_location='right')
hover_UMAP = HoverTool(tooltips="""
<div><span style="font-size: 17px; font-weight: bold;">@Nombre</span></div>
<div><span style="font-size: 12px;">@provincia (@autonomia), @poblacion</span></div>
<div><span style="font-size: 14px; font-weight: bold;">@partido_1</span>
<span style="font-size: 13px;">@porcentaje_1 %</span></div>
<div><span style="font-size: 14px; font-weight: bold;">@partido_2</span>
<span style="font-size: 13px;">@porcentaje_2 %</span></div>
<div><span style="font-size: 14px; font-weight: bold;">@partido_3</span>
<span style="font-size: 13px;">@porcentaje_3 %</span></div>
<div><span style="font-size: 14px; font-weight: bold;">% Abstención</span>
<span style="font-size: 13px;">@porcentaje_abstencion %</span></div>
<div><span style="font-size: 14px; font-weight: bold;">Grupo</span>
<span style="font-size: 13px;">@leiden</span></div>""", )
fig.add_tools(hover_UMAP)
fig.axis.visible, fig.xgrid.visible, fig.ygrid.visible = False, False, False
source = ColumnDataSource(df)
fig.scatter('x',
'y',
source=source,
line_alpha='alpha_line',
line_width=0.3,
line_color="#000000",
size='tamano',
color='color',
alpha='alpha')
return fig, source
UMAP, source = crear_fig()
@panel.depends('obj_opciones', 'obj_autonomias')
def update_df_datos(self):
dict_color_leiden = {
0: '#5F4690',
1: '#1D6996',
2: '#38A6A5',
3: '#0F8554',
4: '#73AF48',
5: '#EDAD08',
6: '#E17C05',
7: '#CC503E',
8: '#94346E',
9: '#6F4070',
10: '#994E95',
11: '#666666',
12: '#11A579',
13: '#E73F74',
14: '#CF1C90',
15: '#3969AC',
16: '#4b4b8f',
17: '#66C5CC',
18: '#F89C74',
19: '#DCB0F2',
20: '#FE88B1',
21: '#661100'
}
dict_num_autonomias = {
'Navarra': 0,
'Canarias': 1,
'Baleares': 2,
'Comunidad Valenciana': 3,
'Asturias': 4,
'Andalucía': 5,
'Aragón': 6,
'Murcia': 7,
'Extremadura': 8,
'Ceuta': 9,
'Cataluña': 10,
'Cantabria': 11,
'Madrid': 12,
'Castilla y León': 13,
'La Rioja': 14,
'Euskadi': 15,
'Melilla': 16,
'Castilla la Mancha': 17,
'Galicia': 18
}
sub_df = df.copy()
# Primero aplicamos el color
attr = self.obj_opciones
if attr in ['partido_1', 'partido_2', 'partido_3']:
sub_df['color'] = [dict_colores[i] for i in sub_df[attr]]
elif attr in [
'porcentaje_blancos', 'porcentaje_nulos',
'porcentaje_abstencion'
]:
max_attr = max(sub_df[attr])
sub_df['color'] = [
self.color_mapper('#000000', "#bbbbbb", i / max_attr)
for i in sub_df[attr]
]
elif attr == 'leiden':
sub_df['color'] = [
dict_color_leiden[i % len(dict_color_leiden)]
for i in sub_df[attr]
]
elif attr == 'provincia':
sub_df['color'] = [
dict_color_leiden[dict_num_prov[i] % 15]
for i in sub_df['provincia']
]
elif attr == 'autonomia':
sub_df['color'] = [
dict_color_leiden[dict_num_autonomias[i] % 19]
for i in sub_df[attr]
]
else:
max_attr = max(sub_df[attr])
sub_df['color'] = np.array([
self.color_mapper(
dict_colores[attr.replace('porcentaje_', '')],
"#f0f0f0", i / max_attr) for i in sub_df[attr]
])
# Ahora aplicamos el alpha de las autonomías
aut_select = self.obj_autonomias
sub_df['alpha'] = [alpha_min] * len(sub_df['alpha'])
sub_df['alpha_line'] = [alpha_min] * len(sub_df['alpha_line'])
sub_df.loc[sub_df['autonomia'].isin(aut_select),
'alpha'] = alpha_max
sub_df.loc[sub_df['autonomia'].isin(aut_select),
'alpha_line'] = 0.45
self.source.stream(sub_df, len(sub_df))
return self.UMAP
class Connectivity_Panel(Interface):
"""
Display lineament collection membership via a network graph.
This panel requires a GeneSetCollection as input.
"""
mapping_selector = param.ListSelector(default=None)
mapping_index_name = param.String(default="Gene", precedence=-1.0)
edge_weight_label = param.String(default=None, precedence=-1.0)
update_network = param.Action(
lambda self: self.param.trigger('update_network'))
def __init__(self, *args, **params):
super().__init__(*args, **params)
if self.lineament_collection is None:
raise Warning("Requires a `GeneSetCollection` input to function.")
if self.mapping_selector is None:
avail_mappings = list(self.lineament_collection.lineaments.keys())
self.param["mapping_selector"].objects = avail_mappings
self.set_param(mapping_selector=avail_mappings)
def build_nx_graph(self, selected_mappings=None, weight=None):
"""Construct the networkx graph object form the selected mappings."""
lcoll = self.lineament_collection
graph = nx.Graph()
gene_mappings = lcoll.as_dict(selected_mappings)
for key, genes in gene_mappings.items():
graph.add_node(key, node_type="Feature")
graph.add_nodes_from(genes, node_type="Gene")
if weight is not None:
subset = self.lineament_collection[key].data.sel(
{self.mapping_index_name: genes})
weights = subset[weight].values
new_edges = [(key, gene, {
"weight": weight
}) for gene, weight in zip(genes, weights)]
else:
new_edges = itertools.product([
key,
], genes)
graph.add_edges_from(new_edges)
return graph
# # Get the union of pairwise-intersections of genes.
# # This will effectively be all genes of connectivity greater than one.
# pw_gene_union = set.union(*[set(x) for x in lcoll.pairwise_intersection(selected_mappings).values()])
#
# for coll_name, coll_genes in lcoll.as_dict(selected_mappings).items():
# # Add the lineament name as a node.
# graph.add_node(coll_name, node_type="Feature")
# # Filter the genes, keep only those that should have more than one edge.
# kept_genes = set.intersection(set(coll_genes), pw_gene_union)
# graph.add_nodes_from(kept_genes, node_type="Gene")
#
# # Now get the genes to be combined into a mega-node. These are genes that
# # should just have an edge to this collection spoke node.
# bundled_genes = list(set.difference(set(coll_genes), kept_genes))
# bundle_node = f"{coll_name} bundle of {len(bundled_genes)} genes"
# graph.add_node(bundle_node, node_type="Bundled Genes", size=len(bundled_genes))
#
# if weight is not None:
# subset = self.lineament_collection[coll_name].data.sel({self.mapping_index_name: list(kept_genes)})
# weights = subset[weight].values
# new_edges = [(coll_name, gene, {"weight": weight}) for gene, weight in zip(kept_genes, weights)]
# else:
# new_edges = list(itertools.product([coll_name, ], kept_genes))
#
# new_edges += [(coll_name, bundle_node)]
# graph.add_edges_from(new_edges)
# return graph
@lru_cache()
def layout_graph(self, selected_mappings, weight=None):
graph = self.build_nx_graph(selected_mappings, weight)
# Construct a circular layout for the collection source nodes.
# mapping_layout = nx.circular_layout(selected_mappings)
layout = nx.layout.kamada_kawai_layout(graph, weight=None)
return graph, layout
@param.depends("update_network")
def view(self):
# Create an immutable tuple of mapping keys so that they can be hashed.
hashable_keys = tuple(self.mapping_selector)
graph, layout = self.layout_graph(hashable_keys,
self.edge_weight_label)
return hv.Graph.from_networkx(graph, positions=layout).opts(
tools=['hover'],
padding=0.2,
height=400,
width=500,
color_index="node_type",
node_size=5,
edge_line_width=.5,
xaxis=None,
yaxis=None)
def panel(self):
controls = pn.Param(self.param)
main_layout = pn.Column("### GeneSet Connectivity Graph",
pn.Row(self.view, controls))
help_layout = generate_help_pane(self)
return pn.Tabs(("UMAP", main_layout), ("Documentation", help_layout))
class OverallParameters(param.Parameterized):
localisation = param.String(default="Jegun", label="")
score = param.Range(default=(0, 250), bounds=(0, 250),)
tout_axes = param.Boolean(True, label="")
interfaces_num = param.ListSelector(label="", default=list(CATEGORIES_INT_NUM_REV.keys()))
infos_num = param.ListSelector(label="", default=list(CATEGORIES_X_INFOS_REV))
comp_admin = param.ListSelector(label="", default=list(CATEGORIES_X_COMP_ADMIN_REV.keys()))
comp_usage_num = param.ListSelector(label="", default=list(CATEGORIES_X_COMP_USAGE_REV.keys()))
point_ref = param.Selector(
default=SELECT[2], objects=SELECT, label="Point de référence",
)
niveau_observation = param.Selector(
default=SELECT[2], objects=SELECT, label="Niveau d'observation",
)
niveau_details = param.Selector(
default=SELECT[2], objects=SELECT, label="Niveau de détail",
)
donnees_infra = param.Action(
lambda x: x, doc="""Données Infra-Communales""", precedence=0.7
)
file_name = param.String(
default="Export_mednum.csv",
doc="""
The filename to save to.""",
)
edit_report = param.Action(
lambda x: x.timestamps.append(dt.datetime.utcnow()),
doc="""Editer un rapport""",
precedence=0.7,
)
tiles = gv.tile_sources.StamenTerrain
df_merged = param.DataFrame()
df_score = param.DataFrame()
def __init__(self, **params):
super(OverallParameters, self).__init__(**params)
interim_data, cont_iris, indice_frag = self.define_paths()
# Merged
output_data_path = interim_data / "add_geom_data_to_merged_data.trc.pqt"
if output_data_path.exists():
import geopandas as gpd
self.df_merged = gpd.read_parquet(output_data_path)
else:
self.df_merged = add_geom_data_to_merged_data(
iris_df(cont_iris), read_merged_data(indice_frag)
)
# Create multindex
self.set_dataframes_indexes()
self.set_dataframes_level()
# Create widgets for indicators
self.define_indices_params()
# Define what is level 0 and level 1 to consider
self.set_entity_levels()
# What is selected in each level
self.get_selected_indice_by_level()
# Define define_searchable_element
self.define_searchable_element()
# Download
self.download = pn.widgets.FileDownload(
label="""Exporter les résultats""",
filename=self.file_name,
callback=self._download_callback,
)
def define_paths(self):
data_path = Path("../data")
if not data_path.exists():
data_path = Path("./data")
if not data_path.exists():
data_path = Path("../../data")
raw_data = data_path / "raw/"
external_data = data_path / "external/"
interim_data = data_path / "interim/"
cont_iris = external_data / "france-geojson" / "contours-iris.geojson"
indice_frag = processed_data / "MERGE_data_clean.csv"
return interim_data, cont_iris, indice_frag
def define_searchable_element(self):
self.seachable_localisation = list(
self.df_merged.index.get_level_values(self.level_0_column_names).unique()
)
def define_indices_params(self):
"""
Create all indices parameters -> Will become a TreeCheckBox or Checkbox
"""
self.g_params = []
for k, widget_opts in TREEVIEW_CHECK_BOX.items():
# Voir si description ne peut être passée
widgets_params = self.create_checkbox_type_widget_params(widget_opts)
self.g_params.append(pn.Param(self.param[k], widgets={k: widgets_params}))
def _download_callback(self):
"""
A FileDownload callback will return a file-like object which can be serialized
and sent to the client.
"""
self.file_name = "Export_%s.csv" % self.point_ref
self.download.filename = self.file_name
sio = io.StringIO()
self.df_score.drop("geometry", axis=1).to_csv(sio, index=False)
sio.seek(0)
return sio
def get_params(self):
paramater_names = [par[0] for par in self.get_param_values()]
return pn.Param(
self.param,
parameters=[par for par in paramater_names if par != "df_merged"],
)
def set_dataframes_level(self):
real_name_level = []
for col in self.df_merged.columns:
if col in CATEGORIES_INDICES.keys():
real_name_level.append((col, CATEGORIES_INDICES[col]))
else:
real_name_level.append((col, col))
self.df_merged.columns = pd.MultiIndex.from_tuples(
real_name_level, names=["variable", "nom"]
)
def set_dataframes_indexes(self):
indexes = list(
set(
list(MAP_COL_WIDGETS["level_0"].values())
+ list(MAP_COL_WIDGETS["level_1"].values())
)
)
self.df_merged.set_index(indexes, inplace=True)
@pn.depends("localisation", "point_ref", "niveau_observation", watch=True)
def set_entity_levels(self):
"""Set the entity levels and point values for this entity.
"""
self.level_0_column, self.level_1_column, self.level_2_column = (
MAP_COL_WIDGETS["level_0"]["index"],
MAP_COL_WIDGETS["level_1"][self.point_ref],
MAP_COL_WIDGETS["level_2"][self.niveau_observation],
)
self.level_0_column_names = MAP_COL_WIDGETS["level_0"]["names"]
self.level_0_value = self.localisation
@pn.depends(
"tout_axes",
"interfaces_num",
"infos_num",
"comp_admin",
"comp_usage_num",
watch=True,
)
def get_selected_indice_by_level(self):
"""get the indices of the selected column
Args:
self ([type]): [description]
Returns:
[type]: [description]
"""
param_values = {k: v for k, v in self.param.get_param_values()}
selected_col = []
for axe, indices in param_values.items():
if axe in TREEVIEW_CHECK_BOX.keys() and indices:
for indice in indices:
try:
selected_col += [CATEGORIES_INDICES_REV[indice]]
except:
pass
self.selected_indices_level_0 = list(set(selected_col))
self.selected_indices_level_1 = list(set(selected_col))
return self.selected_indices_level_0, self.selected_indices_level_1
def create_checkbox_type_widget_params(self, widget_opts):
"""Create dict of widget type and checkbox params .
Args:
widget_opts ([type]): [description]
Returns:
[type]: [description]
"""
if len(widget_opts.items()) > 3:
select_options = [
val["nom"]
for opt, val in widget_opts.items()
if opt not in ["nom", "desc"]
]
descriptions = [
val["desc"]
for opt, val in widget_opts.items()
if opt not in ["nom", "desc"]
]
widget_type = TreeViewCheckBox
widgets_params = {
"type": widget_type,
"select_options": select_options,
"select_all": widget_opts["nom"],
"desc": descriptions,
}
else:
descriptions = widget_opts["desc"]
widget_type = Checkbox
widgets_params = {
"name": widget_opts["nom"],
"type": widget_type,
"value": True,
"desc": descriptions,
}
return widgets_params
def set_real_name(df):
real_name_level = []
for col in df.columns:
if col in CATEGORIES_INDICES.keys():
real_name_level.append((col, CATEGORIES_INDICES[col]))
else:
real_name_level.append((col, col))
return real_name_level
def info_localisation(self):
info_loc = {}
index = self.df_merged.xs(
self.localisation, level=self.level_0_column_names, drop_level=False
).index
ids = index.unique().to_numpy()[0]
names = index.names
for k, v in zip(names, ids):
info_loc[k] = v
return info_loc
def get_indices_properties(self):
indices_properties = {}
tree = copy.deepcopy(TREEVIEW_CHECK_BOX)
for indic_dict in tree.values():
indic_dict.pop("nom", None)
indic_dict.pop("desc", None)
indices_properties.update(indic_dict)
return indices_properties
@pn.depends(
"localisation",
"point_ref",
"niveau_observation",
"tout_axes",
"interfaces_num",
"infos_num",
"comp_admin",
"comp_usage_num",
watch=True,
)
def score_calculation(self):
indices_properties = self.get_indices_properties()
selected_indices = self.selected_indices_level_0
df = self.df_merged.droplevel("nom", axis=1)
info_loc = self.info_localisation()
if selected_indices != []:
selected_indices_aggfunc = {
k: indices_properties[k]["aggfunc"] for k in selected_indices
}
#
map_info = [self.level_0_column_names]
vdims = map_info + selected_indices
# Aggregation selon la fonction specifié (mean, median)
# au niveau level_1_column sur les indice selectionne selected_indices_aggfunc
score_agg_niveau = df.groupby(self.level_1_column).agg(
selected_indices_aggfunc
)
# Select level 25
df_level_2 = df.xs(
info_loc[self.level_2_column],
level=self.level_2_column,
drop_level=False,
)
# Division par l'aggregation sur la zone level_1_column (pondération)
score_niveau = df_level_2[selected_indices].floordiv(score_agg_niveau) * 100
# Dissolution (i.e. agregation geographique) au niveau de découpage souhaité level_0_column
df = df_level_2.dissolve(
by=[self.level_0_column, self.level_0_column_names],
aggfunc=selected_indices_aggfunc,
)
# Score sur les indices merge sur l'index pour récupérer la geometry.
# _BRUT : initial
# _SCORE : Score de l'indice sur le découpage level_0_column divisé par la fonction d'aggragation au level_1_column
scores = df.merge(
score_niveau,
on=[self.level_0_column, self.level_0_column_names],
suffixes=("_BRUT", "_SCORE"),
).drop_duplicates() # Drop duplicate pour supprimer les doublons (zone homogène)
# Calcul des scores sur chaque axes et au total
number_axes = 0
for axe, indices in AXES_INDICES.items():
selected_in_axes = [
k + "_SCORE" for k in indices.keys() if k in selected_indices
]
if selected_in_axes != []:
scores.loc[:, axe] = scores[selected_in_axes].mean(axis=1).astype(int)
number_axes += 1
else:
scores.loc[:, axe] = 0
# Score total
scores.loc[:, "tout_axes"] = scores[list(AXES_INDICES.keys())].sum(axis=1)
if number_axes != 0:
scores.loc[:, "tout_axes"] //= number_axes
self.df_score = df.merge(
scores, on=[self.level_0_column, self.level_0_column_names, "geometry"]
).drop_duplicates() # Suppression des doublons sur les communes découpées en IRIS
else:
df = df.xs(
info_loc[self.level_2_column],
level=self.level_2_column,
drop_level=False,
).dissolve(by=[self.level_0_column, self.level_0_column_names],)
for axe, indices in AXES_INDICES.items():
df.loc[:, axe] = 0
df.loc[:, "tout_axes"] = 0
self.df_score = df
class Q(param.Parameterized):
r = param.ListSelector(default=7, objects=[7, 8])
class KickstarterDashboard(param.Parameterized):
# pylint: disable=line-too-long
"""The purpose of the Kickstarter Dashboard is to test if the claims regarding Bokeh as of Jan 2018 in the
[bokeh-dash-best-dashboard-framework](https://www.sicara.ai/blog/2018-01-30-bokeh-dash-best-dashboard-framework-python)
article holds for Panel and the HoloViews suite of tools as of Dec 2019.
The claims where
- Data in Bokeh becomes inconsistent
- Cannot link charts to dataframe
- Bokeh is slow for big datasets
- Interactions take a long time to develop
You can evaluate this dashboard and the code to make your personal evaluation of the above
statements.
My evaluation is
- the **first two statements does no longer hold**.
- The third is up for discussion. I would also like the Dashboard updates to be a bit faster. Maybe it's because I don't yet know how to implement this efficiently.
- The fourth I've also experienced
see this [discussion](https://discourse.holoviz.org/t/how-to-create-a-parameterized-dashboard-with-seperation-between-data-transforms-and-data-views/53/13).
I can see that I made a lot of mistakes because it takes time for me to understand how the api works.
There is a lot to I need to learn across the HoloViz suite of tools."""
# pylint: enable=line-too-long
kickstarter_df = param.DataFrame()
categories = param.ListSelector()
scatter_df = param.DataFrame()
bar_df = param.DataFrame()
rangexy = param.ClassSelector(
class_=hv.streams.RangeXY,
default=hv.streams.RangeXY(),
)
def __init__(self,
kickstarter_df: Optional[pd.DataFrame] = None,
**kwargs):
if not isinstance(
kickstarter_df,
pd.DataFrame,
):
kickstarter_df = self.get_kickstarter_df()
categories = self.get_categories(kickstarter_df)
self.param.kickstarter_df.default = kickstarter_df
self.param.categories.default = categories
self.param.categories.objects = categories
self.param.scatter_df.default = kickstarter_df
self.param.bar_df.default = kickstarter_df
super().__init__(**kwargs)
@param.depends(
"kickstarter_df",
"categories",
watch=True,
)
def _set_scatter_df(self, ):
self.scatter_df = self.filter_on_categories(
self.kickstarter_df,
self.categories,
)
@param.depends("scatter_df")
def scatter_plot_view(self, ):
"""A Reactive View of the scatter plot"""
# Potential Improvements
# Rename columns to Capitalized without under score
# Add name of movie to hover tooltip
scatter_plot = self.get_scatter_plot(self.scatter_df)
# Please note that depending on how the scatter_plot is generated it might be a Scatter
# or Ndoverlay objects
# In the first case use scatter_plot. In the second case use scatter_plot.last
self.rangexy.source = scatter_plot.last
return scatter_plot
@param.depends(
"scatter_df",
"rangexy.x_range",
"rangexy.y_range",
watch=True,
)
def _set_bar_df(self, ):
"""Update the bar_df dataframe"""
self.bar_df = self.filter_on_ranges(
self.scatter_df,
self.rangexy.x_range, # pylint: disable=no-member
self.rangexy.y_range, # pylint: disable=no-member
)
@param.depends("bar_df")
def bar_chart_view(self, ):
"""A Reactive View of the Bar Chart"""
return self.get_bar_chart(self.bar_df)
def view(self, ):
"""A Reactive View of the KickstarterDashboard"""
pn.config.sizing_mode = "stretch_width"
template = pn.template.FastListTemplate(title="Kickstarter Dashboard",
theme_toggle=False)
template.sidebar[:] = [
pn.pane.HTML("<h2>Settings</h2>"),
pn.Param(
self.param.categories,
widgets={"categories": {
"size": len(self.categories),
}},
),
]
template.main[:] = [
APPLICATION.intro_section(),
pn.Row(
pn.Column(self.scatter_plot_view,
self.bar_chart_view,
sizing_mode="stretch_width"), ),
]
return template
@staticmethod
def _extract() -> pd.DataFrame:
"""Extracts the kickstarter data into a DataFrame
Returns:
pd.DataFrame -- A Dataframe of kickstarter data with
columns=["created_at", "usd_pledged", "state", "category_slug"]
"""
return pd.read_csv(
KICKSTARTER_PATH,
parse_dates=DATE_COLUMNS,
)
@staticmethod
def _transform(
source_data: pd.DataFrame,
n_samples: int = N_SAMPLES,
) -> pd.DataFrame:
"""Transform the data by
- adding broader_category,
- converting usd_pledged to millions
- sampling to n_samples
Arguments:
source_data {pd.DataFrame} -- The source kickstarter data
Returns:
pd.DataFrame -- The transformed DataFrame with
columns=["created_at", "usd_pledged", "state", "category_slug", "broader_category"]
"""
source_data["broader_category"] = source_data[
"category_slug"].str.split("/").str.get(0)
source_data["usd_pledged"] = source_data["usd_pledged"] / 10**6
return source_data.sample(n_samples)
@classmethod
def get_kickstarter_df(cls, ) -> pd.DataFrame:
"""The Dataframe of Kickstarter Data
Returns:
[pd.DataFrame] -- The Dataframe of Kickstarter Data
"""
source_data = cls._extract()
kickstarter_df = cls._transform(source_data)
return kickstarter_df
@staticmethod
def get_categories(kickstarter_df, ) -> List[str]:
"""The list of kickstarter broader categories
Arguments:
kickstarter_df {[type]} -- [description]
Returns:
List[str] -- [description]
"""
return list(kickstarter_df["broader_category"].unique())
@classmethod
def filter_on_categories(
cls,
kickstarter_df: pd.DataFrame,
categories: List[str],
) -> pd.DataFrame:
"""Filters the kickstarter_df by the specified categories
Arguments:
kickstarter_df {pd.DataFrame} -- A Kickstarter Dataframe
categories {List[str]} -- The list of broader_category in the DataFrame
Returns:
pd.DataFrame -- The filtered DataFrame
"""
if categories is None or categories == []:
categories = cls.get_categories(kickstarter_df)
categories_filter = kickstarter_df["broader_category"].isin(categories)
return kickstarter_df[categories_filter]
@staticmethod
def filter_on_ranges(
kickstarter_df: pd.DataFrame,
x_range,
y_range,
) -> pd.DataFrame:
"""Filter the kickstarter_df by x_range and y_range
Arguments:
kickstarter_df {pd.DataFrame} -- [description]
x_range {[type]} -- The usd_pledged range
y_range {[type]} -- The created_at range
Returns:
pd.DataFrame -- The filtered DataFrame
"""
sub_df = kickstarter_df
if y_range:
if not np.isnan(y_range[0]):
sub_df = sub_df[kickstarter_df["usd_pledged"] >= y_range[0]]
if not np.isnan(y_range[1]):
sub_df = sub_df[kickstarter_df["usd_pledged"] <= y_range[1]]
if x_range:
if not np.isnan(x_range[0]):
sub_df = sub_df[kickstarter_df["created_at"] >= x_range[0]]
if not np.isnan(x_range[1]):
sub_df = sub_df[kickstarter_df["created_at"] <= x_range[1]]
return sub_df
@staticmethod
def get_scatter_plot(kickstarter_df: pd.DataFrame, ): # pylint: disable=missing-return-type-doc
"""A Scatter plot of the kickstarter_df
Arguments:
kickstarter_df {pd.DataFrame} -- The DataFrame of kickstarter data
Returns:
[type] -- A Scatter plot
"""
# Potential Improvements
# Rename columns to Capitalized without under score
# Add name of movie to hover tooltip
kickstarter_df["color"] = kickstarter_df["state"]
return kickstarter_df.hvplot.scatter(
x="created_at",
y="usd_pledged",
# color="color",
by="state",
cmap=list(CMAP.values()),
height=400,
responsive=True,
yformatter="%.1fM",
)
@staticmethod
def get_bar_chart(kickstarter_df: pd.DataFrame, ): # pylint: disable=missing-return-type-doc
"""A bar chart of the kickstarter_df
Arguments:
kickstarter_df {pd.DataFrame} -- A DataFrame of Kickstarter data
Returns:
[type] -- A bar chart of the kickstarter_df
"""
# Potential improvements
# Sort by Number of Projects Desc to make it easier to see what large and small
# Filter
stacked_barchart_df = (kickstarter_df[[
"broader_category",
"state",
"created_at",
]].groupby([
"broader_category",
"state",
]).count().rename(columns={"created_at": "Number of projects"}))
# Plot
bar_chart = stacked_barchart_df.hvplot.bar(
stacked=True,
height=400,
responsive=True,
xlabel="Number of projects",
cmap=CMAP,
)
return bar_chart
class Application(param.Parameterized):
pipeline_run_selector = param.ListSelector(default=[], objects=[])
hyperparameter_selector = param.ListSelector(default=[], objects=[])
slicing_metric_selector = param.ObjectSelector(default='', objects=[''])
performance_metric_selector = param.ObjectSelector(objects=[])
def __init__(self, **params):
super(Application, self).__init__(**params)
# lists
result_list = []
hparam_list = []
repo: Repository = Repository.get_instance()
# get all pipelines in this workspace
all_pipelines: List[TrainingPipeline] = repo.get_pipelines_by_type(
[TrainingPipeline.PIPELINE_TYPE])
# get a dataframe of all results + all hyperparameter combinations
for p in all_pipelines:
# This is slowing the comparison down but
# necessary to update the status of each run
if p.get_status() == PipelineStatusTypes.Succeeded.name:
eval_path = p.get_artifacts_uri_by_component(
GDPComponent.Evaluator.name)[0]
evaluation = tfma.load_eval_result(eval_path)
for s, m in evaluation.slicing_metrics:
result_list.append(
dict([('pipeline_name', '{}'.format(p.name)),
('slice_name', s[0][0] if s else ''),
('slice_value', s[0][1] if s else '')]))
result_list[-1].update(
{f'metric_{k}': m[k]['']
for k, v in m.items()})
h_dict = p.get_hyperparameters()
h_dict['pipeline_name'] = p.name
hparam_list.append(h_dict)
self.results = pd.DataFrame([parse_metrics(r) for r in result_list])
self.hparam_info = pd.DataFrame(hparam_list)
# set params
self.param.pipeline_run_selector.objects = self.results[
'pipeline_name'].unique()
@param.depends('pipeline_run_selector', watch=True)
def _updated_context(self):
df = self.results[self.results['pipeline_name'].isin(
self.pipeline_run_selector)]
df = df.dropna(axis=1, how='all')
slicing_metric_list = sorted(list(df['slice_name'].unique()))
performance_metric_set = {
c
for c in df.columns if c.startswith('metric_')
}
performance_metric_list = [None] + sorted(list(performance_metric_set))
self.param['slicing_metric_selector'].objects = slicing_metric_list
self.param[
'performance_metric_selector'].objects = performance_metric_list
# get params
parameter_list = self.hparam_info[
self.hparam_info['pipeline_name'].isin(
self.pipeline_run_selector)].columns
parameter_list = [x for x in parameter_list if x != 'pipeline_name']
self.param['hyperparameter_selector'].objects = parameter_list
self.slicing_metric_selector = ''
self.performance_metric_selector = None
@param.depends('slicing_metric_selector',
'performance_metric_selector',
watch=True)
def performance_graph(self):
if self.performance_metric_selector:
df = self.results[(
self.results['pipeline_name'].isin(self.pipeline_run_selector)
) & (self.results['slice_name'] == self.slicing_metric_selector)]
fig = px.scatter(df,
x='pipeline_name',
y=self.performance_metric_selector,
color='slice_value',
width=1100,
title='Pipeline Comparison')
fig = fig.update_traces(mode='lines+markers')
else:
fig = px.scatter(pd.DataFrame(),
marginal_y='rug',
width=1100,
title='Pipeline Comparison')
return fig
@param.depends('performance_metric_selector',
'hyperparameter_selector',
watch=True)
def parameter_graph(self):
if self.performance_metric_selector and len(
self.hyperparameter_selector) > 0:
df = self.results[(
self.results['pipeline_name'].isin(self.pipeline_run_selector))
& (self.results['slice_name'] == '')]
# merge
extra_df = pd.merge(self.hparam_info,
df,
on='pipeline_name',
how='left')
dimensions = ['pipeline_name'] + self.hyperparameter_selector + \
[self.performance_metric_selector]
new_dims = []
for d in dimensions:
try:
new_dims.append({
'label': d,
'values': pd.to_numeric(extra_df[d])
})
except:
u = sorted(
list(extra_df[d].apply(lambda x: str(x)).unique()))
mapping = {v: i for i, v in enumerate(u)}
new_dims.append({
'label':
d,
'tickvals': [mapping[x] for x in u],
'ticktext':
u,
'values':
extra_df[d].apply(lambda x: str(x)).map(mapping)
})
final_col = pd.to_numeric(
extra_df[self.performance_metric_selector])
fig = go.Figure(data=go.Parcoords(line=dict(color=final_col,
colorscale='inferno',
showscale=True,
cmin=min(final_col),
cmax=max(final_col)),
dimensions=new_dims))
else:
fig = px.scatter(pd.DataFrame(),
marginal_y='rug',
width=1100,
title='Hyperparameter Comparison')
return fig
class Q(param.Parameterized):
r = param.ListSelector(default=6, check_on_set=False)
class Test(param.Parameterized):
a = param.ListSelector(default=['b', 1], objects=[1, 'b', 'c'])
class Q(param.Parameterized):
r = param.ListSelector(default=None, compute_default_fn=lambda: 1)
class HpcConfigurable(param.Parameterized):
configuration_file = param.String()
uit_client = param.ClassSelector(Client)
environment_variables = param.ClassSelector(OrderedDict,
default=OrderedDict())
modules_to_load = param.ListSelector(default=[])
modules_to_unload = param.ListSelector(default=[])
load_modules = param.List()
unload_modules = param.List()
@param.depends('uit_client', watch=True)
def update_configurable_hpc_parameters(self):
if not self.uit_client.connected:
return
self.load_config_file()
self.param.modules_to_unload.objects = sorted(
self.uit_client.get_loaded_modules())
self.param.modules_to_load.objects = self._get_modules_available_to_load(
)
self.modules_to_load = self._validate_modules(
self.param.modules_to_load.objects, self.modules_to_load)
self.unload_modules = self._validate_modules(
self.param.modules_to_unload.objects, self.modules_to_unload)
def _get_modules_available_to_load(self):
modules = set(self.uit_client.get_available_modules(
flatten=True)) - set(self.param.modules_to_unload.objects)
return sorted(modules)
def _validate_modules(self, possible, candidates):
df = pd.DataFrame([v.split('/', 1) for v in possible],
columns=['Name', 'Version'])
df['Default'] = df['Version'].apply(
lambda v: True if v is None else v.endswith('(default)'))
dfg = df.groupby('Name')
modules = list()
for m in candidates:
if m in possible:
modules.append(m)
continue
elif m in dfg.groups:
group = dfg.get_group(m)
row = group.iloc[0]
if group.shape[0] > 1:
row = group[group['Default']].iloc[0]
module = f'{row.Name}/{row.Version}'
modules.append(module)
else:
print(f'Module "{m}" is invalid.')
return sorted(modules)
def load_config_file(self):
config_file = Path(self.configuration_file)
if config_file.is_file():
with config_file.open() as f:
config = yaml.safe_load(f).get(self.uit_client.system, {})
modules = config.get('modules')
if modules:
self.modules_to_load = modules.get(
'load') or self.modules_to_load
self.modules_to_unload = modules.get(
'unload') or self.modules_to_unload
self.environment_variables = OrderedDict(
config.get(
'environment_variables')) or self.environment_variables
class Q(param.Parameterized):
r = param.ListSelector(default=None)
class FileBrowser(param.Parameterized):
"""
"""
path = param.ClassSelector(Path, precedence=-1)
path_text = param.String(label='', precedence=0.3)
home = param.Action(lambda self: self.go_home(), label='🏠', precedence=0.1)
up = param.Action(lambda self: self.move_up(), label='⬆️', precedence=0.2)
refresh_control = param.Action(lambda self: self.validate(),
label='🔄',
precedence=0.25)
callback = param.Action(lambda x: None, precedence=-1)
file_listing = param.ListSelector(
default=[],
label='Single click to select a file or directory:',
precedence=0.5)
patterns = param.List(precedence=-1, default=['*'])
show_hidden = param.Boolean(default=False,
label='Show Hidden Files',
precedence=0.35)
_disabled = param.Boolean(default=False, precedence=-1)
def __init__(self, delayed_init=False, disabled=False, **params):
self.delayed_init = delayed_init
super().__init__(**params)
self.file_listing_widget = None
self._initialize_path()
self.disabled = disabled
def init(self):
self.delayed_init = False
self._initialize_path()
def _initialize_path(self):
if self.delayed_init:
return
if self.path_text:
self.validate()
if not self.path:
self.go_home()
def _new_path(self, path):
return Path(path)
@property
def disabled(self):
return self._disabled
@disabled.setter
def disabled(self, disabled):
for p in self.controls + ['file_listing', 'path_text', 'show_hidden']:
self.param[p].constant = disabled
self._disabled = disabled
@property
def value(self):
if self.file_listing:
return [str(self.path / v) for v in self.file_listing]
else:
return [self.path.as_posix()]
def stop_loading(self):
if self.file_listing_widget is not None:
self.file_listing_widget.css_classes = ['uit-loading']
self.file_listing_widget.css_classes = []
# self.file_listing_widget.param.trigger('css_classes')
def do_callback(self, changed=True):
self.stop_loading()
if self.callback:
self.callback(changed)
def go_home(self):
self.path_text = Path.cwd().as_posix()
self.file_listing = []
self.do_callback()
def move_up(self):
self.path_text = self.path.parent.as_posix()
self.file_listing = []
self.do_callback()
@param.depends('file_listing', watch=True)
def move_down(self):
if self.file_listing:
filename = self.file_listing[0]
fn = self.path / filename
self.path_text = fn.as_posix()
self.do_callback()
@param.depends('path_text', watch=True)
def validate(self):
"""Check that inputted path is valid - set validator accordingly"""
path = self._new_path(self.path_text)
if path and path.is_dir():
self.path = path
self.file_listing = []
elif path and path.is_file():
with param.discard_events(self):
self.path = path.parent
self.file_listing = [path.name]
else:
logger.warning(f'Invalid Directory: {path}')
self.make_options()
@param.depends('show_hidden', watch=True)
def make_options(self):
selected = []
try:
selected = [
p.name + '/' for p in self.path.glob('*') if p.is_dir()
]
for pattern in self.patterns:
selected.extend([
p.name for p in self.path.glob(pattern) if not p.is_dir()
])
if not self.show_hidden:
selected = [p for p in selected if not str(p).startswith('.')]
except Exception as e:
logger.exception(str(e))
self.param.file_listing.objects = sorted(selected)
self.stop_loading()
@property
def controls(self):
return ['home', 'up', 'refresh_control']
@property
def control_styles(self):
styles = {c: {'width': 25} for c in self.controls}
styles.update(path_text={'width_policy': 'max'}, )
return styles
@param.depends('_disabled')
def panel(self):
self.file_listing_widget = pn.widgets.MultiSelect.from_param(
self.param.file_listing, height=200, width_policy='max')
widgets = pn.Param(
self,
parameters=self.controls + ['path_text'],
widgets=self.control_styles,
show_name=False,
)[:]
args = {'listing': self.file_listing_widget}
code = 'listing.css_classes.push("pn-loading", "arcs"); listing.properties.css_classes.change.emit();'
self.file_listing_widget.jscallback(args=args, value=code)
for wg in widgets[:-1]:
wg.js_on_click(args=args, code=code)
widgets[-1].jscallback(args=args, value=code)
return pn.Column(
pn.Row(*widgets, sizing_mode='stretch_width', margin=0),
self.param.show_hidden,
self.file_listing_widget,
sizing_mode='stretch_width',
margin=0,
)
class Q(param.Parameterized):
q = param.ListSelector([5], objects=[4])
class RaHeatMapPlot(HeatMapPlot, param.Parameterized, Reader):
"""
Base class for heat maps with right ascention (RA) on the x-axis.
"""
_cache_flags = Property(proptype=bool, key="cache_flags", default=False)
_cache_reset_time = Property(proptype=int,
key="flag_cache_reset_seconds",
default=86400)
flag_mask = param.Boolean(default=True)
flags = param.ListSelector(
objects=[
"bad_calibration_fpga_restart",
"globalflag",
"acjump",
"acjump_sd",
"rain",
"rain_sd",
"bad_calibration_acquisition_restart",
"misc",
"rain1mm",
"rain1mm_sd",
"srs/bad_ringmap_broadband",
"bad_calibration_gains",
"snow",
"decorrelated_cylinder",
],
default=[
"bad_calibration_fpga_restart",
"acjump_sd",
"bad_calibration_acquisition_restart",
"rain1mm_sd",
"srs/bad_ringmap_broadband",
"bad_calibration_gains",
"snow",
"decorrelated_cylinder",
],
)
def __init__(self,
name: str,
activated: bool = True,
config=None,
**params):
HeatMapPlot.__init__(self, name, activated, **params)
self.read_config(config)
@property
def param_control(self):
"""Overwrite param_control to use MultiChoice widget for the flags selection."""
p = panel.param.Param(
self.param,
expand_button=False,
widgets={
"flags": panel.widgets.MultiChoice,
},
width=325,
)
return panel.Column(p)
def _flags_mask(self, index_map_ra):
if self._cache_flags:
flag_time_spans = get_flags_cached(self.flags,
self._cache_reset_time)
else:
flag_time_spans = get_flags(
self.flags,
csd_to_unix(self.lsd.lsd),
csd_to_unix(self.lsd.lsd + 1),
)
csd_arr = self.lsd.lsd + index_map_ra / 360.0
flag_mask = np.zeros_like(csd_arr, dtype=np.bool)
for type_, ca, cb in flag_time_spans:
flag_mask[(csd_arr > unix_to_csd(ca))
& (csd_arr < unix_to_csd(cb))] = True
return flag_mask[:, np.newaxis]
class ScatterAutonPart(param.Parameterized):
lista_indepe = sorted(list(dict.fromkeys(df[var_indepe].values)))
lista_subordinada = list(dict.fromkeys(df[var_subord].values))
lista_opciones_subordinada = list(dict.fromkeys(df[var_subord].values))
indepe_obj = param.ListSelector(default=lista_indepe, objects=lista_indepe)
subordinada_obj = param.ListSelector(default=lista_subordinada, objects=lista_subordinada)
correlacion_obj = param.Boolean(False, doc="Mostrar correlación")
def crear_plot(self, sub_df, corr):
y_start, y_end = min(sub_df[y][sub_df['alpha'] == alpha_max]), max(sub_df[y][sub_df['alpha'] == alpha_max])
y_start -= 0.05 * (y_end - y_start)
y_end += 0.05 * (y_end - y_start)
scatter = figure(plot_height=400, plot_width=400,
tools='reset,box_zoom,pan,wheel_zoom,lasso_select,undo,redo',
sizing_mode='scale_width', output_backend="webgl", toolbar_location='above',
y_range=(y_start, y_end))
hover = HoverTool(
tooltips="""
<div><span style="font-size: 17px; font-weight: bold;">@municipio</span></div>
<div><span style="font-size: 12px;">@provincia (@autonomia), @poblacion</span></div>
<div><span style="font-size: 14px; font-weight: bold;">@partido</span>
<span style="font-size: 12px;">@porcentaje</span></div>
""")
scatter.add_tools(hover)
scatter.scatter(x=x, y=y, source=sub_df, color='color', alpha='alpha', **scatter_kwargs)
y_range=(y_start, y_end)
if corr: # Con esto añadimos las correlaciones.
for var_subor_i in self.subordinada_obj:
for var_indep_i in self.indepe_obj:
si_df = sub_df[(sub_df[var_indepe] == var_indep_i) &
(sub_df[var_subord] == var_subor_i)]
if len(si_df) > 1: # Nos aseguramos porque si no falla para Ceuta y Melilla
x_vals, y_vals = si_df[x].values, si_df[y].values
if corr_type == 'lineal':
def f(x, m, b):
return m * x + b
m, b, r, p, err = lr(x_vals, y_vals)
text_label = "r² = %.2f" % r**2
elif corr_type == 'exp':
def f(x, m, b):
return np.power(m * np.log10(x) + b, 10)
popt, pcor = cf(f, x_vals, y_vals)
m, b = popt
ss_res = np.sum((y_vals - f(x_vals, m, b)) ** 2)
ss_tot = np.sum((y_vals - np.mean(y_vals)) ** 2)
r_squared = 1 - (ss_res / ss_tot)
text_label = "r² = %.2f" % r_squared
x_arr = np.linspace(min(x_vals), max(x_vals), 100)
scatter.line(x_arr, [f(x_i, m, b) for x_i in x_arr], color=si_df['color'].iloc[0])
r_label = Label(x=1.05 * max(x_vals), y=f(max(x_vals), m, b),
text=text_label, text_align='left', text_color=si_df['color'].iloc[0],
render_mode='css')
scatter.add_layout(r_label)
if f(max(x_vals), m, b) > y_end: y_end = f(max(x_vals), m, b)
if f(max(x_vals), m, b) < y_start: y_start = f(max(x_vals), m, b)
else:
pass
scatter.y_range = Range1d(y_start, y_end)
return scatter
@panel.depends('indepe_obj', watch=True)
def actualizar_subordinada(self):
sub_df = df[df[var_indepe].isin(self.indepe_obj)]
lista_subordinada = list(dict.fromkeys(sub_df[var_subord].values))
self.param['subordinada_obj'].objects = lista_subordinada
self.param['subordinada_obj'].default = lista_subordinada
@panel.depends('indepe_obj', 'subordinada_obj', 'correlacion_obj')
def plotear_auton(self):
sub_df = df[df[var_indepe].isin(self.indepe_obj)]
sub_df['alpha'] = alpha_min
sub_df.loc[sub_df[var_subord].isin(self.subordinada_obj), 'alpha'] = alpha_max
si = self.correlacion_obj
return self.crear_plot(sub_df, si)