def make_demo(width=20, ball_radius=5, path_radius=5, steps=30):
x, y = la.Axis(width, 'x'), la.Axis(width, 'y')
t = la.Axis(steps, 't')
center = (width - 1) / 2
ball_center_x = la.sin(la.radians(t * 360 / steps)) * path_radius + center
ball_center_y = la.cos(la.radians(t * 360 / steps)) * path_radius + center
return la.maximum(
ball_radius - la.sqrt((x - ball_center_x)**2 + (y - ball_center_y)**2),
0).transpose(x, y)
def prepare(self, args, kwargs):
axes = [la.Axis(np.unique(arg)) for arg in args]
c = kwargs.get('c', 'b')
unq_colors = np.unique(c)
if len(unq_colors) >= self.colorbar_threshold:
# we will add a colorbar in this case, so we do not need a legend
self.show_legend = False
else:
# prepend a fake axis that will be used to make a legend
axes = [la.Axis(unq_colors)] + axes
return args, axes
def prepare(self, args, kwargs):
ndim_req = self.ndim_req
dimerror = ValueError("%s() only works on %d or %d dimensional data" %
(self.funcname, ndim_req - 1, ndim_req))
if self.check_length and len(args) > 1:
if all(np.isscalar(a) for a in args):
args = [np.asarray(args)]
else:
length = len(args[0])
if any(len(a) != length for a in args):
raise ValueError(
"when plotting multiple arrays, they must "
"have compatible axes")
if len(args) == 1:
data = args[0]
if not isinstance(data, (np.ndarray, la.LArray)):
data = np.asarray(data)
if ndim(data) == ndim_req:
# move the last axis first so that the last dimension is stacked
axes = list(range(data.ndim))
data = data.transpose(axes[-1], *axes[:-1])
elif ndim(data) == ndim_req - 1:
if isinstance(data, la.LArray):
# add dummy axis and move it as the first axis
data = data.expand(la.Axis(
1, '__dummy__')).transpose('__dummy__')
else:
data = data[np.newaxis]
else:
raise dimerror
elif all(ndim(a) == ndim_req - 1 for a in args):
data = args
else:
raise dimerror
return np.asarray(data), aslabeledarray(data).axes
def _setup_and_check(self, widget, data, title, readonly, **kwargs):
"""
Setup ArrayComparator.
Parameters
----------
widget: QWidget
Parent widget.
data: list or tuple of Array, ndarray
Arrays to compare.
title: str
Title.
readonly: bool
kwargs:
* rtol: int or float
* atol: int or float
* nans_equal: bool
* bg_gradient: str
* names: list of str
"""
arrays = [
la.asarray(array) for array in data
if isinstance(array, DISPLAY_IN_GRID)
]
names = kwargs.get('names',
["Array{}".format(i) for i in range(len(arrays))])
layout = QVBoxLayout()
widget.setLayout(layout)
comparator_widget = ComparatorWidget(self, **kwargs)
comparator_widget.set_data(arrays, la.Axis(names, 'array'))
layout.addWidget(comparator_widget)
def _from_selection(self, raw_data, axes_names, vlabels, hlabels):
if '\\' in axes_names[-1]:
axes_names = axes_names[:-1] + axes_names[-1].split('\\')
if len(axes_names) == 2:
axes = [la.Axis(vlabels[0], axes_names[0])]
elif len(axes_names) > 2:
# combine the N-1 first axes
combined_axes_names = '_'.join(axes_names[:-1])
combined_labels = ['_'.join([str(vlabels[i][j]) for i in range(len(vlabels))])
for j in range(len(vlabels[0]))]
axes = [la.Axis(combined_labels, combined_axes_names)]
else:
# assuming selection represents a 1D array
axes = []
raw_data = raw_data[0]
# last axis
axes += [la.Axis(hlabels, axes_names[-1])]
return la.Array(raw_data, axes)
def aslabeledarray(data):
sequence = (tuple, list)
if isinstance(data, la.LArray):
return data
elif (isinstance(data, sequence) and len(data)
and isinstance(data[0], la.LArray)):
# XXX: use la.stack?
# TODO: check that all arrays have the same axes
axes = [la.Axis(len(data))] + list(data[0].axes)
return la.LArray(data, axes)
else:
return la.LArray(data)
else:
_orig_display_hook(value)
def install_display_hook():
sys.displayhook = _qt_display_hook
def restore_display_hook():
sys.displayhook = _orig_display_hook
if __name__ == "__main__":
"""Array editor test"""
lipro = la.Axis(['P%02d' % i for i in range(1, 16)], 'lipro')
age = la.Axis('age=0..115')
sex = la.Axis('sex=M,F')
vla = 'A11,A12,A13,A23,A24,A31,A32,A33,A34,A35,A36,A37,A38,A41,A42,' \
'A43,A44,A45,A46,A71,A72,A73'
wal = 'A25,A51,A52,A53,A54,A55,A56,A57,A61,A62,A63,A64,A65,A81,A82,' \
'A83,A84,A85,A91,A92,A93'
bru = 'A21'
# list of strings
belgium = la.union(vla, wal, bru)
geo = la.Axis(belgium, 'geo')
# data1 = np.arange(30).reshape(2, 15)
# arr1 = la.LArray(data1, axes=(sex, lipro))
def _setup_and_check(self, widget, data, title, readonly, **kwargs):
"""
Setup SessionComparator.
Parameters
----------
widget: QWidget
Parent widget.
data: list or tuple of Session
Sessions to compare.
title: str
Title.
readonly: bool
kwargs:
* rtol: int or float
* atol: int or float
* nans_equal: bool
* bg_gradient: str
* names: list of str
* colors: str
"""
sessions = data
names = kwargs.get(
'names', ["Session{}".format(i) for i in range(len(sessions))])
assert all(isinstance(s, la.Session) for s in sessions)
self.sessions = sessions
self.stack_axis = la.Axis(names, 'session')
layout = QVBoxLayout()
widget.setLayout(layout)
array_names = sorted(
set.union(*[
set(s.filter(kind=DISPLAY_IN_GRID).names)
for s in self.sessions
]))
listwidget = QListWidget(self)
listwidget.addItems(array_names)
listwidget.currentItemChanged.connect(self.on_item_changed)
for i, name in enumerate(array_names):
arrays = self.get_arrays(name)
first_array = arrays[0]
if not all(
a.equals(first_array, nans_equal=True)
for a in arrays[1:]):
listwidget.item(i).setForeground(Qt.red)
self.listwidget = listwidget
comparatorwidget = ComparatorWidget(self, **kwargs)
self.arraywidget = comparatorwidget
main_splitter = QSplitter(Qt.Horizontal)
main_splitter.addWidget(listwidget)
main_splitter.addWidget(comparatorwidget)
main_splitter.setSizes([5, 95])
main_splitter.setCollapsible(1, False)
self.widget_state_settings['main_splitter'] = main_splitter
layout.addWidget(main_splitter)
self.listwidget.setCurrentRow(0)
def make_sphere(width=20, radius=9):
x, y, z = la.Axis(width, 'x'), la.Axis(width, 'y'), la.Axis(width, 'z')
center = (width - 1) / 2
return la.maximum(
radius - la.sqrt((x - center)**2 + (y - center)**2 + (z - center)**2),
0)
def compute(self, context, *expressions, **kwargs):
if not expressions:
raise TypeError("groupby() takes at least 1 argument")
# TODO: allow lists/tuples of arguments to group by the combinations
# of keys
for expr in expressions:
if isinstance(expr, (bool, int, float)):
raise TypeError("groupby() does not work with constant "
"arguments")
if isinstance(expr, (tuple, list)):
raise TypeError("groupby() takes expressions as arguments, "
"not a list of expressions")
# On python 3, we could clean up this code (keyword only arguments).
expr = kwargs.pop('expr', None)
if expr is None:
expr = Count()
# by = kwargs.pop('by', None)
filter_value = kwargs.pop('filter', None)
percent = kwargs.pop('percent', False)
possible_values = kwargs.pop('pvalues', None)
totals = kwargs.pop('totals', True)
expr_vars = [v.name for v in collect_variables(expr)]
labels = [str(e) for e in expressions]
columns = [expr_eval(e, context) for e in expressions]
columns = [expand(c, context_length(context)) for c in columns]
if filter_value is not None:
filtered_columns = [col[filter_value] for col in columns]
# FIXME: use the actual filter_expr instead of not_hashable
filtered_context = context.subset(filter_value, expr_vars,
not_hashable)
else:
filtered_columns = columns
filtered_context = context
if possible_values is None:
possible_values = [np.unique(col) for col in filtered_columns]
# We pre-filtered columns instead of passing the filter to partition_nd
# because it is a bit faster this way. The indices are still correct,
# because we use them on a filtered_context.
groups = partition_nd(filtered_columns, True, possible_values)
if not groups:
# return la.LArray([], labels, possible_values)
return la.LArray([])
# evaluate the expression on each group
# we use not_hashable to avoid storing the subset in the cache
contexts = [
filtered_context.subset(indices, expr_vars, not_hashable)
for indices in groups
]
data = [expr_eval(expr, c) for c in contexts]
# TODO: use group_indices_nd directly to avoid using np.unique
# this is twice as fast (unique is very slow) but breaks because
# the rest of the code assumes all combinations are present
# if self.filter is not None:
# filter_value = expr_eval(self.filter, context)
# else:
# filter_value = True
#
# d = group_indices_nd(columns, filter_value)
# pvalues = sorted(d.keys())
# ndim = len(columns)
# possible_values = [[pv[i] for pv in pvalues]
# for i in range(ndim)]
# groups = [d[k] for k in pvalues]
# groups is a (flat) list of list.
# the first variable is the outer-most "loop",
# the last one the inner most.
# add total for each row
len_pvalues = [len(vals) for vals in possible_values]
if percent:
totals = True
if totals:
width = len_pvalues[-1]
height = prod(len_pvalues[:-1])
rows_indices = [
np.concatenate([groups[y * width + x] for x in range(width)])
for y in range(height)
]
cols_indices = [
np.concatenate([groups[y * width + x] for y in range(height)])
for x in range(width)
]
cols_indices.append(np.concatenate(cols_indices))
# evaluate the expression on each "combined" group (ie compute totals)
row_ctxs = [
filtered_context.subset(indices, expr_vars, not_hashable)
for indices in rows_indices
]
row_totals = [expr_eval(expr, ctx) for ctx in row_ctxs]
col_ctxs = [
filtered_context.subset(indices, expr_vars, not_hashable)
for indices in cols_indices
]
col_totals = [expr_eval(expr, ctx) for ctx in col_ctxs]
else:
row_totals = None
col_totals = None
if percent:
# convert to np.float64 to get +-inf if total_value is int(0)
# instead of Python's built-in behaviour of raising an exception.
# This can happen at least when using the default expr (count())
# and the filter yields empty groups
total_value = np.float64(col_totals[-1])
data = [100.0 * value / total_value for value in data]
row_totals = [100.0 * value / total_value for value in row_totals]
col_totals = [100.0 * value / total_value for value in col_totals]
# if self.by or self.percent:
# if self.percent:
# total_value = data[-1]
# divisors = [total_value for _ in data]
# else:
# num_by = len(self.by)
# inc = prod(len_pvalues[-num_by:])
# num_groups = len(groups)
# num_categories = prod(len_pvalues[:-num_by])
#
# categories_groups_idx = [range(cat_idx, num_groups, inc)
# for cat_idx in range(num_categories)]
#
# divisors = ...
#
# data = [100.0 * value / divisor
# for value, divisor in zip(data, divisors)]
# convert to a 1d array. We don't simply use data = np.array(data),
# because if data is a list of ndarray (for example if we use
# groupby(a, expr=id), *and* all the ndarrays have the same length,
# the result is a 2d array instead of an array of ndarrays like we
# need (at this point).
arr = np.empty(len(data), dtype=type(data[0]))
arr[:] = data
data = arr
# and reshape it
data = data.reshape(len_pvalues)
axes = [
la.Axis(axis_labels, axis_name)
for axis_name, axis_labels in zip(labels, possible_values)
]
# FIXME: also handle totals
return la.LArray(data, axes)
def index_tables(globals_def, entities, fpath):
print("reading data from %s ..." % fpath)
input_file = tables.open_file(fpath)
try:
input_root = input_file.root
def must_load_from_input_file(gdef):
return isinstance(gdef, dict) and 'path' not in gdef
any_global_from_input_file = any(
must_load_from_input_file(gdef) for gdef in globals_def.values())
if any_global_from_input_file and 'globals' not in input_root:
raise Exception(
'could not find any globals in the input data file '
'(but some are declared in the simulation file)')
globals_data = load_path_globals(globals_def)
constant_globals_data = handle_constant_globals(globals_def)
globals_data.update(constant_globals_data)
globals_node = getattr(input_root, 'globals', None)
for name, global_def in globals_def.items():
# already loaded from another source (path)
if name in globals_data:
continue
if name not in globals_node:
raise Exception("could not find 'globals/%s' in the input "
"data file" % name)
global_data = getattr(globals_node, name)
global_type = global_def.get('type', global_def.get('fields'))
# TODO: move the checking (assertValidType) to a separate function
assert_valid_type(global_data, global_type, context=name)
array = global_data.read()
if isinstance(global_type, list):
# make sure we do not keep in memory columns which are
# present in the input file but where not asked for by the
# modeller. They are not accessible anyway.
array = add_and_drop_fields(array, global_type)
attrs = global_data.attrs
dim_names = getattr(attrs, 'dimensions', None)
if dim_names is not None:
# we serialise dim_names as a numpy array so that it is
# stored as a native hdf type and not a pickle but we
# prefer to work with simple lists
# also files serialized using Python2 are "bytes" not "str"
dim_names = [str(dim_name) for dim_name in dim_names]
pvalues = [
getattr(attrs, 'dim%d_pvalues' % i)
for i in range(len(dim_names))
]
axes = [
la.Axis(labels, axis_name)
for axis_name, labels in zip(dim_names, pvalues)
]
array = la.LArray(array, axes)
globals_data[name] = array
input_entities = input_root.entities
entities_tables = {}
print(" * indexing tables")
for ent_name, entity in entities.items():
print(" -", ent_name, "...", end=' ')
table = getattr(input_entities, ent_name)
assert_valid_type(table, list(entity.fields.in_input.name_types))
rows_per_period, id_to_rownum_per_period = \
timed(index_table, table)
indexed_table = IndexedTable(table, rows_per_period,
id_to_rownum_per_period)
entities_tables[ent_name] = indexed_table
except:
input_file.close()
raise
return input_file, {'globals': globals_data, 'entities': entities_tables}