def lightgbm_model(self, target, features=None, num_boost_round=100, copy=False, params={}, prediction_name='lightgbm_prediction'): '''Requires vaex.ml: create a lightgbm model and train/fit it. :param target: The name of the target column. :param features: List of features to use when training the model. If None, all columns except the target will be used as features. :param num_boost_round: Number of boosting rounds. :param bool copy: If True Copy the data, otherwise use a more memory efficient data transfer method. :return vaex.ml.lightgbm.LightGBMModel: Fitted LightGBM model. ''' from .lightgbm import LightGBMModel dataframe = self.df target = _ensure_strings_from_expressions(target) features = features or self.df.get_column_names( virtual=True).remove(target) features = _ensure_strings_from_expressions(features) booster = LightGBMModel(prediction_name=prediction_name, num_boost_round=num_boost_round, features=features, target=target, params=params) booster.fit(dataframe, copy=copy) return booster
def catboost_model(self, target, features=None, num_boost_round=100, params=None, prediction_name='catboost_prediction'): '''Requires vaex.ml: create a CatBoostModel model and train/fit it. :param target: The name of the target column. :param features: List of features to use when training the model. If None, all columns except the target will be used as features. :param num_boost_round: Number of boosting rounds. :return vaex.ml.catboost.CatBoostModel: Fitted CatBoostModel model. ''' from .catboost import CatBoostModel dataframe = self.df target = _ensure_strings_from_expressions(target) features = features or self.df.get_column_names( virtual=True).remove(target) features = _ensure_strings_from_expressions(features) booster = CatBoostModel(prediction_name=prediction_name, num_boost_round=num_boost_round, features=features, target=target, params=params) booster.fit(dataframe) return booster
def expand(self, stop=[]): stop = _ensure_strings_from_expressions(stop) def translate(id): if id in self.ds.virtual_columns and id not in stop: return self.ds.virtual_columns[id] expr = expresso.translate(self.expression, translate) return Expression(self.ds, expr)
def pygbm_model(self, label, max_iter, features=None, param={}, classifier=False, prediction_name='pygbm_prediction', **kwargs): '''Requires vaex.ml: create a pygbm model and train/fit it. :param label: Label to train/fit on :param max_iter: Max number of iterations/trees :param features: List of features to train on :param bool classifier: If true, return a the classifier (will use argmax on the probabilities) :return vaex.ml.pygbm.PyGBMModel or vaex.ml.pygbm.PyGBMClassifier: Fitted PyGBM model ''' from .incubator.pygbm import PyGBMModel, PyGBMClassifier dataframe = self.df features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) cls = PyGBMClassifier if classifier else PyGBMModel b = cls(prediction_name=prediction_name, max_iter=max_iter, features=features, param=param, **kwargs) b.fit(dataframe, label) return b
def expand(self, stop=[]): stop = _ensure_strings_from_expressions(stop) def translate(id): if id in self.ds.virtual_columns and id not in stop: return self.ds.virtual_columns[id] expr = expresso.translate(self.expression, translate) return Expression(self.ds, expr)
def lightgbm_model(self, target, num_boost_round, features=None, copy=False, params={}, prediction_name='lightgbm_prediction'): '''Requires vaex.ml: create a lightgbm model and train/fit it. :param target: The target variable to predict. :param num_boost_round: Number of boosting iterations. :param features: List of features to train on. :param bool copy: Copy data or use the modified xgboost library for efficient transfer. :return vaex.ml.lightgbm.LightGBMModel: Fitted LightGBM model. ''' from .lightgbm import LightGBMModel dataframe = self.df features = features or self.df.get_column_names(virtual=True) features = _ensure_strings_from_expressions(features) booster = LightGBMModel(prediction_name=prediction_name, num_boost_round=num_boost_round, features=features, params=params) booster.fit(dataframe, target, copy=copy) return booster
def iter_vaex( X: vaex.dataframe.DataFrame, y: typing.Union[str, vaex.expression.Expression] = None, features: typing.Union[typing.List[str], vaex.expression.Expression] = None, ) -> base.typing.Stream: """Yields rows from a ``vaex.DataFrame``. Parameters ---------- X A vaex DataFrame housing the training featuers. y The column or expression containing the target variable. features A list of features used for training. If None, all columns in `X` will be used. Features specifying in `y` are ignored. """ features = _ensure_strings_from_expressions(features) feature_names = features or X.get_column_names() if y: y = _ensure_strings_from_expressions(y) y = _ensure_list(y) feature_names = [feat for feat in feature_names if feat not in y] multioutput = len(y) > 1 if multioutput: for i in range(len(X)): yield ( {key: X.evaluate(key, i, i + 1)[0] for key in feature_names}, {key: X.evaluate(key, i, i + 1)[0] for key in y}, ) else: for i in range(len(X)): yield ( {key: X.evaluate(key, i, i + 1)[0] for key in feature_names}, X.evaluate(y[0], i, i + 1)[0], )
def label_encoder(self, features=None, prefix='label_encoded_'): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.LabelEncoder` and fit it. :param features: List of features to encode. :param prefix: Prefix for the names of the encoded features. ''' features = features or self.get_column_names() features = _ensure_strings_from_expressions(features) label_encoder = LabelEncoder(features=features, prefix=prefix) label_encoder.fit(self) return label_encoder
def iter_vaex(X, y=None, features=None, **kwargs): """Yields rows from a ``vaex.DataFrame``. Parameters: X (vaex.DataFrame): A vaex DataFrame housing the training featuers. y (string or vaex.Expression): The column or expression containing the target variable. features (list of strings or vaex.Expressions): A list of features used for training. If None, all columns in ``X`` will be used. Features specifying in ``y`` are ignored. Yields: tuple: A pair (``x``, ``y``) where ``x`` is a dict of features and ``y`` is the target. """ from vaex.utils import _ensure_strings_from_expressions, _ensure_list features = _ensure_strings_from_expressions(features) feature_names = features or X.get_column_names() if y: y = _ensure_strings_from_expressions(y) y = _ensure_list(y) feature_names = [feat for feat in feature_names if feat not in y] multioutput = len(y) > 1 if multioutput: for i in range(len(X)): yield ({ key: X.evaluate(key, i, i + 1)[0] for key in feature_names }, {key: X.evaluate(key, i, i + 1)[0] for key in y}) else: for i in range(len(X)): yield ({ key: X.evaluate(key, i, i + 1)[0] for key in feature_names }, X.evaluate(y[0], i, i + 1)[0])
def minmax_scaler(self, features=None, feature_range=[0, 1], prefix='minmax_scaled_'): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.MinMaxScaler` and fit it. :param features: List of features to scale. :param feature_range: The range the features are scaled to. :param prefix: Prefix for the names of the scaled features. ''' features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) minmax_scaler = MinMaxScaler(features=features, feature_range=feature_range, prefix=prefix) minmax_scaler.fit(self.df) return minmax_scaler
def label_encoder(self, features=None, prefix='label_encoded_', allow_unseen=False): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.LabelEncoder` and fit it. :param features: List of features to encode. :param prefix: Prefix for the names of the encoded features. :param allow_unseen: If True, encode unseen value as -1, otherwise an error is raised. ''' features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) label_encoder = LabelEncoder(features=features, prefix=prefix, allow_unseen=allow_unseen) label_encoder.fit(self.df) return label_encoder
def standard_scaler(self, features=None, with_mean=True, with_std=True, prefix='standard_scaled_'): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.StandardScaler` and fit it. :param features: List of features to scale. :param with_mean: If True, remove the mean from each feature. :param with_std: If True, scale each feature to unit variance. :param prefix: Prefix for the names of the scaled features. ''' features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) standard_scaler = StandardScaler(features=features, with_mean=with_mean, with_std=with_std, prefix=prefix) standard_scaler.fit(self.df) return standard_scaler
def pca(self, n_components=2, features=None, prefix='PCA_', progress=False): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.PCA` and fit it. :param n_components: Number of components to retain. If None, all the components will be retained. :param features: List of features to transform. :param prefix: Prefix for the names of the transformed features. :param progress: If True, display a progressbar of the PCA fitting process. ''' features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) pca = PCA(n_components=n_components, features=features, progress=progress) pca.fit(self.df) return pca
def frequency_encoder(self, features=None, unseen='nan', prefix='frequency_encoded_'): ''' Requires vaex.ml: Create :class:`vaex.ml.transformations.FrequencyEncoder` and fit it. :param features: List of features to encode. :param unseen: Strategy to deal with unseen values. Accepted arguments are "zero" or "nan". :param prefix: Prefix for the names of the encoded features. ''' features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) freq_encoder = FrequencyEncoder(features=features, prefix=prefix) freq_encoder.fit(self.df) return freq_encoder
def fit(self, df): '''Fit OneHotEncoder to the DataFrame. :param df: A vaex DataFrame. ''' uniques = [] for i in self.features: expression = _ensure_strings_from_expressions(i) unique = df.unique(expression) unique = np.sort(unique) # this can/should be optimized with @delay uniques.append(unique.tolist()) self.uniques_ = uniques
def test_py_gbm_virtual_columns(): ds = vaex.ml.datasets.load_iris() ds['x'] = ds.sepal_length * 1 ds['y'] = ds.sepal_width * 1 ds['w'] = ds.petal_length * 1 ds['z'] = ds.petal_width * 1 ds_train, ds_test = ds.ml.train_test_split(test_size=0.2, verbose=False) features = ['x', 'y', 'z', 'w'] booster = vaex.ml.incubator.pygbm.PyGBMModel( num_round=10, param=param, features=_ensure_strings_from_expressions(features)) booster.fit(ds_train, ds_train.class_)
def one_hot_encoder(self, features=None, one=1, zero=0, prefix=''): '''Requires vaex.ml: Create :class:`vaex.ml.transformations.OneHotEncoder` and fit it. :param features: List of features to encode. :param one: What value to use instead of "1". :param zero: What value to use instead of "0". :param prefix: Prefix for the names of the encoded features. :returns one_hot_encoder: vaex.ml.transformations.OneHotEncoder object. ''' if features is None: raise ValueError('Please give at least one categorical feature.') features = _ensure_strings_from_expressions(features) one_hot_encoder = OneHotEncoder(features=features, one=one, zero=zero, prefix=prefix) one_hot_encoder.fit(self.df) return one_hot_encoder
def test_lightgbm(): ds = vaex.ml.datasets.load_iris() ds_train, ds_test = ds.ml.train_test_split(test_size=0.2, verbose=False) features = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] features = _ensure_strings_from_expressions(features) booster = vaex.ml.lightgbm.LightGBMModel(num_boost_round=10, params=params, features=features, target='class_') booster.fit(ds_train, copy=True) # for coverage class_predict_train = booster.predict(ds_train, copy=True) # for coverage class_predict_test = booster.predict(ds_test) assert np.all(ds_test.col.class_.values == np.argmax(class_predict_test, axis=1)) ds_train = booster.transform(ds_train) # this will add the lightgbm_prediction column state = ds_train.state_get() ds_test.state_set(state) assert np.all(ds_test.col.class_.values == np.argmax(ds_test.lightgbm_prediction.values, axis=1))
def expand(self, stop=[]): """Expand the expression such that no virtual columns occurs, only normal columns. Example: >>> df = vaex.example() >>> r = np.sqrt(df.data.x**2 + df.data.y**2) >>> r.expand().expression 'sqrt(((x ** 2) + (y ** 2)))' """ stop = _ensure_strings_from_expressions(stop) def translate(id): if id in self.ds.virtual_columns and id not in stop: return self.ds.virtual_columns[id] expr = expresso.translate(self.ast, translate) return Expression(self.ds, expr)
def wrapper(self, features=None, transform=True, **kwargs): kwargs = kwargs.copy( ) # we do modifications, so make a copy features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) import importlib module = importlib.import_module(class_spec['module']) cls = getattr(module, class_spec['classname']) if 'target' in kwargs: kwargs['target'] = str(kwargs['target']) object = cls(features=features, **kwargs) object.fit(self.df) if transform: dft = object.transform(self.df) return dft else: return object
def evaluate(self, expression, i1=None, i2=None, out=None, selection=None, delay=False): expression = _ensure_strings_from_expressions(expression) """basic support for evaluate at server, at least to run some unittest, do not expect this to work from strings""" result = self.server._call_dataset("evaluate", self, expression=expression, i1=i1, i2=i2, selection=selection, delay=delay) # TODO: we ignore out return result
def test_lightgbm(): ds = vaex.ml.datasets.load_iris() ds_train, ds_test = ds.ml.train_test_split(test_size=0.2, verbose=False) features = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] booster = vaex.ml.lightgbm.LightGBMModel( num_boost_round=10, params=params, features=_ensure_strings_from_expressions(features)) booster.fit(ds_train, ds.class_, copy=True) # for coverage class_predict = booster.predict(ds_test, copy=True) # for coverage booster.fit(ds_train, ds.class_) class_predict = booster.predict(ds_test) assert np.all(ds.col.class_ == class_predict) ds = booster.transform(ds) # this will add the lightgbm_prediction column state = ds.state_get() ds = vaex.ml.datasets.load_iris() ds.state_set(state) assert np.all(ds.col.class_ == ds.evaluate(ds.lightgbm_prediction))
def fit(self, df): '''Fit OneHotEncoder to the DataFrame. :param df: A vaex DataFrame. ''' uniques = [] for i in self.features: expression = _ensure_strings_from_expressions(i) unique_values = vaex.array_types.tolist(df.unique(expression)) if None in unique_values: unique_values.remove(None) unique_values.sort() unique_values.insert(0, None) # This is done in place else: unique_values.sort() uniques.append(unique_values) self.uniques_ = uniques
def test_pygbm(): for filename in 'blah.col.meta blah.col.page'.split(): if os.path.exists(filename): os.remove(filename) ds = vaex.ml.datasets.load_iris() ds_train, ds_test = ds.ml.train_test_split(test_size=0.2, verbose=False) features = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] booster = vaex.ml.incubator.pygbm.PyGBMModel( num_round=10, param=param, features=_ensure_strings_from_expressions(features)) booster.fit(ds_train, ds_train.class_) class_predict = booster.predict(ds_test) booster.fit(ds_train, ds_train.class_) class_predict = booster.predict(ds_test) assert np.all(ds.col.class_ == class_predict) ds = booster.transform(ds) # this will add the pygbm_prediction column state = ds.state_get() ds = vaex.ml.datasets.load_iris() ds.state_set(state) assert np.all(ds.col.class_ == ds.evaluate(ds.pygbm_prediction))
def xgboost_model(self, target, num_boost_round, features=None, params={}, prediction_name='xgboost_prediction'): '''Requires vaex.ml: create a XGBoost model and train/fit it. :param target: Target to train/fit on. :param num_boost_round: Number of rounds. :param features: List of features to train on. :return vaex.ml.xgboost.XGBModel: Fitted XGBoost model. ''' from .xgboost import XGBoostModel dataframe = self features = features or self.get_column_names() features = _ensure_strings_from_expressions(features) booster = XGBoostModel(prediction_name=prediction_name, num_boost_round=num_boost_round, features=features, params=params) booster.fit(dataframe, target) return booster
def catboost_model(self, target, num_boost_round, features=None, params=None, prediction_name='catboost_prediction'): '''Requires vaex.ml: create a CatBoostModel model and train/fit it. :param target: Target to train/fit on :param num_boost_round: Number of rounds :param features: List of features to train on :return vaex.ml.catboost.CatBoostModel: Fitted CatBoostModel model ''' from .catboost import CatBoostModel dataframe = self.df features = features or self.df.get_column_names() features = _ensure_strings_from_expressions(features) booster = CatBoostModel(prediction_name=prediction_name, num_boost_round=num_boost_round, features=features, params=params) booster.fit(dataframe, target) return booster
def plot( self, x=None, y=None, z=None, what="count(*)", vwhat=None, reduce=["colormap"], f=None, normalize="normalize", normalize_axis="what", vmin=None, vmax=None, shape=256, vshape=32, limits=None, grid=None, colormap="afmhot", # colors=["red", "green", "blue"], figsize=None, xlabel=None, ylabel=None, aspect="auto", tight_layout=True, interpolation="nearest", show=False, colorbar=True, colorbar_label=None, selection=None, selection_labels=None, title=None, background_color="white", pre_blend=False, background_alpha=1., visual=dict(x="x", y="y", layer="z", fade="selection", row="subspace", column="what"), smooth_pre=None, smooth_post=None, wrap=True, wrap_columns=4, return_extra=False, hardcopy=None): """Viz data in a 2d histogram/heatmap. Declarative plotting of statistical plots using matplotlib, supports subplots, selections, layers. Instead of passing x and y, pass a list as x argument for multiple panels. Give what a list of options to have multiple panels. When both are present then will be origanized in a column/row order. This methods creates a 6 dimensional 'grid', where each dimension can map the a visual dimension. The grid dimensions are: * x: shape determined by shape, content by x argument or the first dimension of each space * y: ,, * z: related to the z argument * selection: shape equals length of selection argument * what: shape equals length of what argument * space: shape equals length of x argument if multiple values are given By default, this its shape is (1, 1, 1, 1, shape, shape) (where x is the last dimension) The visual dimensions are * x: x coordinate on a plot / image (default maps to grid's x) * y: y ,, (default maps to grid's y) * layer: each image in this dimension is blended togeher to one image (default maps to z) * fade: each image is shown faded after the next image (default mapt to selection) * row: rows of subplots (default maps to space) * columns: columns of subplot (default maps to what) All these mappings can be changes by the visual argument, some examples: >>> df.plot('x', 'y', what=['mean(x)', 'correlation(vx, vy)']) Will plot each 'what' as a column. >>> df.plot('x', 'y', selection=['FeH < -3', '(FeH >= -3) & (FeH < -2)'], visual=dict(column='selection')) Will plot each selection as a column, instead of a faded on top of each other. :param x: Expression to bin in the x direction (by default maps to x), or list of pairs, like [['x', 'y'], ['x', 'z']], if multiple pairs are given, this dimension maps to rows by default :param y: y (by default maps to y) :param z: Expression to bin in the z direction, followed by a :start,end,shape signature, like 'FeH:-3,1:5' will produce 5 layers between -10 and 10 (by default maps to layer) :param what: What to plot, count(*) will show a N-d histogram, mean('x'), the mean of the x column, sum('x') the sum, std('x') the standard deviation, correlation('vx', 'vy') the correlation coefficient. Can also be a list of values, like ['count(x)', std('vx')], (by default maps to column) :param reduce: :param f: transform values by: 'identity' does nothing 'log' or 'log10' will show the log of the value :param normalize: normalization function, currently only 'normalize' is supported :param normalize_axis: which axes to normalize on, None means normalize by the global maximum. :param vmin: instead of automatic normalization, (using normalize and normalization_axis) scale the data between vmin and vmax to [0, 1] :param vmax: see vmin :param shape: shape/size of the n-D histogram grid :param limits: list of [[xmin, xmax], [ymin, ymax]], or a description such as 'minmax', '99%' :param grid: if the binning is done before by yourself, you can pass it :param colormap: matplotlib colormap to use :param figsize: (x, y) tuple passed to pylab.figure for setting the figure size :param xlabel: :param ylabel: :param aspect: :param tight_layout: call pylab.tight_layout or not :param colorbar: plot a colorbar or not :param interpolation: interpolation for imshow, possible options are: 'nearest', 'bilinear', 'bicubic', see matplotlib for more :param return_extra: :return: """ import pylab import matplotlib n = _parse_n(normalize) if type(shape) == int: shape = (shape, ) * 2 binby = [] x = _ensure_strings_from_expressions(x) y = _ensure_strings_from_expressions(y) for expression in [y, x]: if expression is not None: binby = [expression] + binby fig = pylab.gcf() if figsize is not None: fig.set_size_inches(*figsize) import re what_units = None whats = _ensure_list(what) selections = _ensure_list(selection) selections = _ensure_strings_from_expressions(selections) if y is None: waslist, [ x, ] = vaex.utils.listify(x) else: waslist, [x, y] = vaex.utils.listify(x, y) x = list(zip(x, y)) limits = [limits] # every plot has its own vwhat for now vwhats = _expand_limits(vwhat, len(x)) # TODO: we're abusing this function.. logger.debug("x: %s", x) limits, shape = self.limits(x, limits, shape=shape) shape = shape[0] logger.debug("limits: %r", limits) # mapping of a grid axis to a label labels = {} shape = _expand_shape(shape, 2) vshape = _expand_shape(shape, 2) if z is not None: match = re.match("(.*):(.*),(.*),(.*)", z) if match: groups = match.groups() import ast z_expression = groups[0] logger.debug("found groups: %r", list(groups)) z_limits = [ ast.literal_eval(groups[1]), ast.literal_eval(groups[2]) ] z_shape = ast.literal_eval(groups[3]) # for pair in x: x = [[z_expression] + list(k) for k in x] limits = np.array([[z_limits] + list(k) for k in limits]) shape = (z_shape, ) + shape vshape = (z_shape, ) + vshape logger.debug("x = %r", x) values = np.linspace(z_limits[0], z_limits[1], num=z_shape + 1) labels["z"] = list([ "%s <= %s < %s" % (v1, z_expression, v2) for v1, v2 in zip(values[:-1], values[1:]) ]) else: raise ValueError( "Could not understand 'z' argument %r, expected something in form: 'column:-1,10:5'" % facet) else: z_shape = 1 # z == 1 if z is None: total_grid = np.zeros((len(x), len(whats), len(selections), 1) + shape, dtype=float) total_vgrid = np.zeros( (len(x), len(whats), len(selections), 1) + vshape, dtype=float) else: total_grid = np.zeros((len(x), len(whats), len(selections)) + shape, dtype=float) total_vgrid = np.zeros((len(x), len(whats), len(selections)) + vshape, dtype=float) logger.debug("shape of total grid: %r", total_grid.shape) axis = dict(plot=0, what=1, selection=2) xlimits = limits grid_axes = dict(x=-1, y=-2, z=-3, selection=-4, what=-5, subspace=-6) visual_axes = dict(x=-1, y=-2, layer=-3, fade=-4, column=-5, row=-6) # visual_default=dict(x="x", y="y", z="layer", selection="fade", subspace="row", what="column") # visual: mapping of a plot axis, to a grid axis visual_default = dict(x="x", y="y", layer="z", fade="selection", row="subspace", column="what") def invert(x): return dict((v, k) for k, v in x.items()) # visual_default_reverse = invert(visual_default) # visual_ = visual_default # visual = dict(visual) # copy for modification # add entries to avoid mapping multiple times to the same axis free_visual_axes = list(visual_default.keys()) # visual_reverse = invert(visual) logger.debug("1: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual.items(): if visual_name in free_visual_axes: free_visual_axes.remove(visual_name) else: raise ValueError("visual axes %s used multiple times" % visual_name) logger.debug("2: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual_default.items(): if visual_name in free_visual_axes and grid_name not in visual.values( ): free_visual_axes.remove(visual_name) visual[visual_name] = grid_name logger.debug("3: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual_default.items(): if visual_name not in free_visual_axes and grid_name not in visual.values( ): visual[free_visual_axes.pop(0)] = grid_name logger.debug("4: %r %r", visual, free_visual_axes) visual_reverse = invert(visual) # TODO: the meaning of visual and visual_reverse is changed below this line, super confusing visual, visual_reverse = visual_reverse, visual # so now, visual: mapping of a grid axis to plot axis # visual_reverse: mapping of a grid axis to plot axis move = {} for grid_name, visual_name in visual.items(): if visual_axes[visual_name] in visual.values(): index = visual.values().find(visual_name) key = visual.keys()[index] raise ValueError( "trying to map %s to %s while, it is already mapped by %s" % (grid_name, visual_name, key)) move[grid_axes[grid_name]] = visual_axes[visual_name] # normalize_axis = _ensure_list(normalize_axis) fs = _expand(f, total_grid.shape[grid_axes[normalize_axis]]) # assert len(vwhat) # labels["y"] = ylabels what_labels = [] if grid is None: grid_of_grids = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): grid_of_grids.append([]) for j, what in enumerate(whats): if isinstance(what, vaex.stat.Expression): grid = what.calculate(self, binby=binby, shape=shape, limits=limits, selection=selections, delay=True) else: what = what.strip() index = what.index("(") import re groups = re.match("(.*)\((.*)\)", what).groups() if groups and len(groups) == 2: function = groups[0] arguments = groups[1].strip() if "," in arguments: arguments = arguments.split(",") functions = [ "mean", "sum", "std", "var", "correlation", "covar", "min", "max", "median_approx" ] unit_expression = None if function in [ "mean", "sum", "std", "min", "max", "median" ]: unit_expression = arguments if function in ["var"]: unit_expression = "(%s) * (%s)" % (arguments, arguments) if function in ["covar"]: unit_expression = "(%s) * (%s)" % arguments if unit_expression: unit = self.unit(unit_expression) if unit: what_units = unit.to_string('latex_inline') if function in functions: grid = getattr(self, function)(arguments, binby=binby, limits=limits, shape=shape, selection=selections, delay=True) elif function == "count": grid = self.count(arguments, binby, shape=shape, limits=limits, selection=selections, delay=True) else: raise ValueError( "Could not understand method: %s, expected one of %r'" % (function, functions)) else: raise ValueError( "Could not understand 'what' argument %r, expected something in form: 'count(*)', 'mean(x)'" % what) if i == 0: # and j == 0: what_label = str(whats[j]) if what_units: what_label += " (%s)" % what_units if fs[j]: what_label = fs[j] + " " + what_label what_labels.append(what_label) grid_of_grids[-1].append(grid) self.executor.execute() for i, (binby, limits) in enumerate(zip(x, xlimits)): for j, what in enumerate(whats): grid = grid_of_grids[i][j].get() total_grid[i, j, :, :] = grid[:, None, ...] labels["what"] = what_labels else: dims_left = 6 - len(grid.shape) total_grid = np.broadcast_to(grid, (1, ) * dims_left + grid.shape) # visual=dict(x="x", y="y", selection="fade", subspace="facet1", what="facet2",) def _selection_name(name): if name in [None, False]: return "selection: all" elif name in ["default", True]: return "selection: default" else: return "selection: %s" % name if selection_labels is None: labels["selection"] = list([_selection_name(k) for k in selections]) else: labels["selection"] = selection_labels # visual_grid = np.moveaxis(total_grid, move.keys(), move.values()) # np.moveaxis is in np 1.11 only?, use transpose axes = [None] * len(move) for key, value in move.items(): axes[value] = key visual_grid = np.transpose(total_grid, axes) logger.debug("grid shape: %r", total_grid.shape) logger.debug("visual: %r", visual.items()) logger.debug("move: %r", move) logger.debug("visual grid shape: %r", visual_grid.shape) xexpressions = [] yexpressions = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): xexpressions.append(binby[0]) yexpressions.append(binby[1]) if xlabel is None: xlabels = [] ylabels = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): if z is not None: xlabels.append(self.label(binby[1])) ylabels.append(self.label(binby[2])) else: xlabels.append(self.label(binby[0])) ylabels.append(self.label(binby[1])) else: Nl = visual_grid.shape[visual_axes['row']] xlabels = _expand(xlabel, Nl) ylabels = _expand(ylabel, Nl) #labels[visual["x"]] = (xlabels, ylabels) labels["x"] = xlabels labels["y"] = ylabels # grid = total_grid # print(grid.shape) # grid = self.reduce(grid, ) axes = [] # cax = pylab.subplot(1,1,1) background_color = np.array( matplotlib.colors.colorConverter.to_rgb(background_color)) # if grid.shape[axis["selection"]] > 1:# and not facet: # rgrid = vaex.image.fade(rgrid) # finite_mask = np.any(finite_mask, axis=0) # do we really need this # print(rgrid.shape) # facet_row_axis = axis["what"] import math facet_columns = None facets = visual_grid.shape[visual_axes["row"]] * visual_grid.shape[ visual_axes["column"]] if visual_grid.shape[visual_axes["column"]] == 1 and wrap: facet_columns = min(wrap_columns, visual_grid.shape[visual_axes["row"]]) wrapped = True elif visual_grid.shape[visual_axes["row"]] == 1 and wrap: facet_columns = min(wrap_columns, visual_grid.shape[visual_axes["column"]]) wrapped = True else: wrapped = False facet_columns = visual_grid.shape[visual_axes["column"]] facet_rows = int(math.ceil(facets / facet_columns)) logger.debug("facet_rows: %r", facet_rows) logger.debug("facet_columns: %r", facet_columns) # if visual_grid.shape[visual_axes["row"]] > 1: # and not wrap: # #facet_row_axis = axis["what"] # facet_columns = visual_grid.shape[visual_axes["column"]] # else: # facet_columns = min(wrap_columns, facets) # if grid.shape[axis["plot"]] > 1:# and not facet: # this loop could be done using axis arguments everywhere # assert len(normalize_axis) == 1, "currently only 1 normalization axis supported" grid = visual_grid * 1. fgrid = visual_grid * 1. ngrid = visual_grid * 1. # colorgrid = np.zeros(ngrid.shape + (4,), float) # print "norma", normalize_axis, visual_grid.shape[visual_axes[visual[normalize_axis]]] vmins = _expand(vmin, visual_grid.shape[visual_axes[visual[normalize_axis]]], type=list) vmaxs = _expand(vmax, visual_grid.shape[visual_axes[visual[normalize_axis]]], type=list) # for name in normalize_axis: visual_grid if smooth_pre: grid = vaex.grids.gf(grid, smooth_pre) if 1: axis = visual_axes[visual[normalize_axis]] for i in range(visual_grid.shape[axis]): item = [ slice(None, None, None), ] * len(visual_grid.shape) item[axis] = i item = tuple(item) f = _parse_f(fs[i]) with np.errstate(divide='ignore', invalid='ignore' ): # these are fine, we are ok with nan's in vaex fgrid.__setitem__(item, f(grid.__getitem__(item))) # print vmins[i], vmaxs[i] if vmins[i] is not None and vmaxs[i] is not None: nsubgrid = fgrid.__getitem__(item) * 1 nsubgrid -= vmins[i] nsubgrid /= (vmaxs[i] - vmins[i]) else: nsubgrid, vmin, vmax = n(fgrid.__getitem__(item)) vmins[i] = vmin vmaxs[i] = vmax # print " ", vmins[i], vmaxs[i] ngrid.__setitem__(item, nsubgrid) if 0: # TODO: above should be like the code below, with custom vmin and vmax grid = visual_grid[i] f = _parse_f(fs[i]) fgrid = f(grid) finite_mask = np.isfinite(grid) finite_mask = np.any(finite_mask, axis=0) if vmin is not None and vmax is not None: ngrid = fgrid * 1 ngrid -= vmin ngrid /= (vmax - vmin) ngrid = np.clip(ngrid, 0, 1) else: ngrid, vmin, vmax = n(fgrid) # vmin, vmax = np.nanmin(fgrid), np.nanmax(fgrid) # every 'what', should have its own colorbar, check if what corresponds to # rows or columns in facets, if so, do a colorbar per row or per column rows, columns = int(math.ceil(facets / float(facet_columns))), facet_columns colorbar_location = "individual" if visual["what"] == "row" and visual_grid.shape[1] == facet_columns: colorbar_location = "per_row" if visual["what"] == "column" and visual_grid.shape[0] == facet_rows: colorbar_location = "per_column" # values = np.linspace(facet_limits[0], facet_limits[1], facet_count+1) logger.debug("rows: %r, columns: %r", rows, columns) import matplotlib.gridspec as gridspec column_scale = 1 row_scale = 1 row_offset = 0 if facets > 1: if colorbar_location == "per_row": column_scale = 4 gs = gridspec.GridSpec(rows, columns * column_scale + 1) elif colorbar_location == "per_column": row_offset = 1 row_scale = 4 gs = gridspec.GridSpec(rows * row_scale + 1, columns) else: gs = gridspec.GridSpec(rows, columns) facet_index = 0 fs = _expand(f, len(whats)) colormaps = _expand(colormap, len(whats)) # row for i in range(visual_grid.shape[0]): # column for j in range(visual_grid.shape[1]): if colorbar and colorbar_location == "per_column" and i == 0: norm = matplotlib.colors.Normalize(vmins[j], vmaxs[j]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[j]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[0, j]) colorbar = fig.colorbar(sm, cax=ax, orientation="horizontal") else: colorbar = fig.colorbar(sm) if "what" in labels: label = labels["what"][j] if facets > 1: colorbar.ax.set_title(label) else: colorbar.ax.set_ylabel(colorbar_label or label) if colorbar and colorbar_location == "per_row" and j == 0: norm = matplotlib.colors.Normalize(vmins[i], vmaxs[i]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[i]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[i, -1]) colorbar = fig.colorbar(sm, cax=ax) else: colorbar = fig.colorbar(sm) label = labels["what"][i] colorbar.ax.set_ylabel(colorbar_label or label) rgrid = ngrid[i, j] * 1. # print rgrid.shape for k in range(rgrid.shape[0]): for l in range(rgrid.shape[0]): if smooth_post is not None: rgrid[k, l] = vaex.grids.gf(rgrid, smooth_post) if visual["what"] == "column": what_index = j elif visual["what"] == "row": what_index = i else: what_index = 0 if visual[normalize_axis] == "column": normalize_index = j elif visual[normalize_axis] == "row": normalize_index = i else: normalize_index = 0 for r in reduce: r = _parse_reduction(r, colormaps[what_index], []) rgrid = r(rgrid) row = facet_index // facet_columns column = facet_index % facet_columns if colorbar and colorbar_location == "individual": # visual_grid.shape[visual_axes[visual[normalize_axis]]] norm = matplotlib.colors.Normalize(vmins[normalize_index], vmaxs[normalize_index]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[what_index]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[row, column]) colorbar = fig.colorbar(sm, ax=ax) else: colorbar = fig.colorbar(sm) label = labels["what"][what_index] colorbar.ax.set_ylabel(colorbar_label or label) if facets > 1: ax = pylab.subplot( gs[row_offset + row * row_scale:row_offset + (row + 1) * row_scale, column * column_scale:(column + 1) * column_scale]) else: ax = pylab.gca() axes.append(ax) logger.debug("rgrid: %r", rgrid.shape) plot_rgrid = rgrid assert plot_rgrid.shape[1] == 1, "no layers supported yet" plot_rgrid = plot_rgrid[:, 0] if plot_rgrid.shape[0] > 1: plot_rgrid = vaex.image.fade(plot_rgrid[::-1]) else: plot_rgrid = plot_rgrid[0] extend = None if visual["subspace"] == "row": subplot_index = i elif visual["subspace"] == "column": subplot_index = j else: subplot_index = 0 extend = np.array(xlimits[subplot_index][-2:]).flatten() # extend = np.array(xlimits[i]).flatten() logger.debug("plot rgrid: %r", plot_rgrid.shape) plot_rgrid = np.transpose(plot_rgrid, (1, 0, 2)) im = ax.imshow(plot_rgrid, extent=extend.tolist(), origin="lower", aspect=aspect, interpolation=interpolation) # v1, v2 = values[i], values[i+1] def label(index, label, expression): if label and _issequence(label): return label[i] else: return self.label(expression) if visual_reverse["x"] == 'x': labelsx = labels['x'] pylab.xlabel(labelsx[subplot_index]) if visual_reverse["x"] == 'x': labelsy = labels['y'] pylab.ylabel(labelsy[subplot_index]) if visual["z"] in ['row']: labelsz = labels['z'] ax.set_title(labelsz[i]) if visual["z"] in ['column']: labelsz = labels['z'] ax.set_title(labelsz[j]) max_labels = 10 xexpression = xexpressions[i] if self.iscategory(xexpression): labels = self.category_labels(xexpression) step = max(len(labels) // max_labels, 1) pylab.xticks(np.arange(len(labels))[::step], labels[::step], size='small') yexpression = yexpressions[i] if self.iscategory(yexpression): labels = self.category_labels(yexpression) step = max(len(labels) // max_labels, 1) pylab.yticks(np.arange(len(labels))[::step], labels[::step], size='small') facet_index += 1 if title: fig.suptitle(title, fontsize="x-large") if tight_layout: if title: pylab.tight_layout(rect=[0, 0.03, 1, 0.95]) else: pylab.tight_layout() if hardcopy: pylab.savefig(hardcopy) if show: pylab.show() if return_extra: return im, grid, fgrid, ngrid, rgrid else: return im
def plot1d(self, x=None, what="count(*)", grid=None, shape=64, facet=None, limits=None, figsize=None, f="identity", n=None, normalize_axis=None, xlabel=None, ylabel=None, label=None, selection=None, show=False, tight_layout=True, hardcopy=None, progress=None, **kwargs): """Viz data in 1d (histograms, running means etc) Example >>> df.plot1d(df.x) >>> df.plot1d(df.x, limits=[0, 100], shape=100) >>> df.plot1d(df.x, what='mean(y)', limits=[0, 100], shape=100) If you want to do a computation yourself, pass the grid argument, but you are responsible for passing the same limits arguments: >>> counts = df.mean(df.y, binby=df.x, limits=[0, 100], shape=100)/100. >>> df.plot1d(df.x, limits=[0, 100], shape=100, grid=means, label='mean(y)/100') :param x: Expression to bin in the x direction :param what: What to plot, count(*) will show a N-d histogram, mean('x'), the mean of the x column, sum('x') the sum :param grid: If the binning is done before by yourself, you can pass it :param facet: Expression to produce facetted plots ( facet='x:0,1,12' will produce 12 plots with x in a range between 0 and 1) :param limits: list of [xmin, xmax], or a description such as 'minmax', '99%' :param figsize: (x, y) tuple passed to pylab.figure for setting the figure size :param f: transform values by: 'identity' does nothing 'log' or 'log10' will show the log of the value :param n: normalization function, currently only 'normalize' is supported, or None for no normalization :param normalize_axis: which axes to normalize on, None means normalize by the global maximum. :param normalize_axis: :param xlabel: String for label on x axis (may contain latex) :param ylabel: Same for y axis :param: tight_layout: call pylab.tight_layout or not :param kwargs: extra argument passed to pylab.plot :return: """ import pylab f = _parse_f(f) n = _parse_n(n) if type(shape) == int: shape = (shape, ) binby = [] x = _ensure_strings_from_expressions(x) for expression in [x]: if expression is not None: binby = [expression] + binby limits = self.limits(binby, limits) if figsize is not None: pylab.figure(num=None, figsize=figsize, dpi=80, facecolor='w', edgecolor='k') fig = pylab.gcf() import re if facet is not None: match = re.match("(.*):(.*),(.*),(.*)", facet) if match: groups = match.groups() facet_expression = groups[0] facet_limits = [ ast.literal_eval(groups[1]), ast.literal_eval(groups[2]) ] facet_count = ast.literal_eval(groups[3]) limits.append(facet_limits) binby.append(facet_expression) shape = (facet_count, ) + shape else: raise ValueError( "Could not understand 'facet' argument %r, expected something in form: 'column:-1,10:5'" % facet) if grid is None: if what: if isinstance(what, (vaex.stat.Expression)): grid = what.calculate(self, binby=binby, limits=limits, shape=shape, selection=selection) else: what = what.strip() index = what.index("(") import re groups = re.match("(.*)\((.*)\)", what).groups() if groups and len(groups) == 2: function = groups[0] arguments = groups[1].strip() functions = ["mean", "sum", "std", "count"] if function in functions: # grid = getattr(self, function)(arguments, binby, limits=limits, shape=shape, selection=selection) grid = getattr(vaex.stat, function)(arguments).calculate( self, binby=binby, limits=limits, shape=shape, selection=selection, progress=progress) elif function == "count" and arguments == "*": grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection, progress=progress) elif function == "cumulative" and arguments == "*": # TODO: comulative should also include the tails outside limits grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection, progress=progress) grid = np.cumsum(grid) else: raise ValueError( "Could not understand method: %s, expected one of %r'" % (function, functions)) else: raise ValueError( "Could not understand 'what' argument %r, expected something in form: 'count(*)', 'mean(x)'" % what) else: grid = self.histogram(binby, size=shape, limits=limits, selection=selection) fgrid = f(grid) if n is not None: # ngrid = n(fgrid, axis=normalize_axis) ngrid = fgrid / fgrid.sum() else: ngrid = fgrid # reductions = [_parse_reduction(r, colormap, colors) for r in reduce] # rgrid = ngrid * 1. # for r in reduce: # r = _parse_reduction(r, colormap, colors) # rgrid = r(rgrid) # grid = self.reduce(grid, ) xmin, xmax = limits[-1] if facet: N = len(grid[-1]) else: N = len(grid) xexpression = binby[0] xar = np.arange(N + 1) / (N - 0.) * (xmax - xmin) + xmin label = str(label or selection or x) if facet: import math rows, columns = int(math.ceil(facet_count / 4.)), 4 values = np.linspace(facet_limits[0], facet_limits[1], facet_count + 1) for i in range(facet_count): ax = pylab.subplot(rows, columns, i + 1) value = ax.plot(xar, ngrid[i], drawstyle="steps-mid", label=label, **kwargs) v1, v2 = values[i], values[i + 1] pylab.xlabel(xlabel or x) pylab.ylabel(ylabel or what) ax.set_title("%3f <= %s < %3f" % (v1, facet_expression, v2)) if self.iscategory(xexpression): labels = self.category_labels(xexpression) step = len(labels) // max_labels pylab.xticks(range(len(labels))[::step], labels[::step], size='small') else: # im = pylab.imshow(rgrid, extent=np.array(limits[:2]).flatten(), origin="lower", aspect=aspect) pylab.xlabel(xlabel or self.label(x)) pylab.ylabel(ylabel or what) # print(xar, ngrid) # repeat the first element, that's how plot/steps likes it.. g = np.concatenate([ngrid[0:1], ngrid]) value = pylab.plot(xar, g, drawstyle="steps-pre", label=label, **kwargs) if self.iscategory(xexpression): labels = self.category_labels(xexpression) step = len(labels) // max_labels pylab.xticks(range(len(labels))[::step], labels[::step], size='small') if tight_layout: pylab.tight_layout() if hardcopy: pylab.savefig(hardcopy) if show: pylab.show() return value
def scatter(self, x, y, xerr=None, yerr=None, cov=None, corr=None, s_expr=None, c_expr=None, labels=None, selection=None, length_limit=50000, length_check=True, label=None, xlabel=None, ylabel=None, errorbar_kwargs={}, ellipse_kwargs={}, **kwargs): """Viz (small amounts) of data in 2d using a scatter plot Convenience wrapper around pylab.scatter when for working with small DataFrames or selections :param x: Expression for x axis :param y: Idem for y :param s_expr: When given, use if for the s (size) argument of pylab.scatter :param c_expr: When given, use if for the c (color) argument of pylab.scatter :param labels: Annotate the points with these text values :param selection: Single selection expression, or None :param length_limit: maximum number of rows it will plot :param length_check: should we do the maximum row check or not? :param label: label for the legend :param xlabel: label for x axis, if None .label(x) is used :param ylabel: label for y axis, if None .label(y) is used :param errorbar_kwargs: extra dict with arguments passed to plt.errorbar :param kwargs: extra arguments passed to pylab.scatter :return: """ import pylab as plt x = _ensure_strings_from_expressions(x) y = _ensure_strings_from_expressions(y) label = str(label or selection) selection = _ensure_strings_from_expressions(selection) if length_check: count = self.count(selection=selection) if count > length_limit: raise ValueError( "the number of rows (%d) is above the limit (%d), pass length_check=False, or increase length_limit" % (count, length_limit)) x_values = self.evaluate(x, selection=selection) y_values = self.evaluate(y, selection=selection) if s_expr: kwargs["s"] = self.evaluate(s_expr, selection=selection) if c_expr: kwargs["c"] = self.evaluate(c_expr, selection=selection) plt.xlabel(xlabel or self.label(x)) plt.ylabel(ylabel or self.label(y)) s = plt.scatter(x_values, y_values, label=label, **kwargs) if labels: label_values = self.evaluate(labels, selection=selection) for i, label_value in enumerate(label_values): plt.annotate(label_value, (x_values[i], y_values[i])) xerr_values = None yerr_values = None if cov is not None or corr is not None: from matplotlib.patches import Ellipse sx = self.evaluate(xerr, selection=selection) sy = self.evaluate(yerr, selection=selection) if corr is not None: sxy = self.evaluate(corr, selection=selection) * sx * sy elif cov is not None: sxy = self.evaluate(cov, selection=selection) cov_matrix = np.zeros((len(sx), 2, 2)) cov_matrix[:, 0, 0] = sx**2 cov_matrix[:, 1, 1] = sy**2 cov_matrix[:, 0, 1] = cov_matrix[:, 1, 0] = sxy ax = plt.gca() ellipse_kwargs = dict(ellipse_kwargs) ellipse_kwargs['facecolor'] = ellipse_kwargs.get('facecolor', 'none') ellipse_kwargs['edgecolor'] = ellipse_kwargs.get('edgecolor', 'black') for i in range(len(sx)): eigen_values, eigen_vectors = np.linalg.eig(cov_matrix[i]) indices = np.argsort(eigen_values)[::-1] eigen_values = eigen_values[indices] eigen_vectors = eigen_vectors[:, indices] v1 = eigen_vectors[:, 0] v2 = eigen_vectors[:, 1] varx = cov_matrix[i, 0, 0] vary = cov_matrix[i, 1, 1] angle = np.arctan2(v1[1], v1[0]) # round off errors cause negative values? if eigen_values[1] < 0 and abs( (eigen_values[1] / eigen_values[0])) < 1e-10: eigen_values[1] = 0 if eigen_values[0] < 0 or eigen_values[1] < 0: raise ValueError('neg val') width, height = np.sqrt(np.max(eigen_values)), np.sqrt( np.min(eigen_values)) e = Ellipse(xy=(x_values[i], y_values[i]), width=width, height=height, angle=np.degrees(angle), **ellipse_kwargs) ax.add_artist(e) else: if xerr is not None: if _issequence(xerr): assert len( xerr ) == 2, "if xerr is a sequence it should be of length 2" xerr_values = [ self.evaluate(xerr[0], selection=selection), self.evaluate(xerr[1], selection=selection) ] else: xerr_values = self.evaluate(xerr, selection=selection) if yerr is not None: if _issequence(yerr): assert len( yerr ) == 2, "if yerr is a sequence it should be of length 2" yerr_values = [ self.evaluate(yerr[0], selection=selection), self.evaluate(yerr[1], selection=selection) ] else: yerr_values = self.evaluate(yerr, selection=selection) if xerr_values is not None or yerr_values is not None: errorbar_kwargs = dict(errorbar_kwargs) errorbar_kwargs['fmt'] = errorbar_kwargs.get('fmt', 'none') plt.errorbar(x_values, y_values, yerr=yerr_values, xerr=xerr_values, **errorbar_kwargs) return s
def evaluate(self, expression, i1=None, i2=None, out=None, selection=None, delay=False): expression = _ensure_strings_from_expressions(expression) """basic support for evaluate at server, at least to run some unittest, do not expect this to work from strings""" result = self.server._call_dataset("evaluate", self, expression=expression, i1=i1, i2=i2, selection=selection, delay=delay) # TODO: we ignore out return result
def scatter(self, x, y, xerr=None, yerr=None, cov=None, corr=None, s_expr=None, c_expr=None, labels=None, selection=None, length_limit=50000, length_check=True, label=None, xlabel=None, ylabel=None, errorbar_kwargs={}, ellipse_kwargs={}, **kwargs): """Viz (small amounts) of data in 2d using a scatter plot Convenience wrapper around pylab.scatter when for working with small DataFrames or selections :param x: Expression for x axis :param y: Idem for y :param s_expr: When given, use if for the s (size) argument of pylab.scatter :param c_expr: When given, use if for the c (color) argument of pylab.scatter :param labels: Annotate the points with these text values :param selection: Single selection expression, or None :param length_limit: maximum number of rows it will plot :param length_check: should we do the maximum row check or not? :param label: label for the legend :param xlabel: label for x axis, if None .label(x) is used :param ylabel: label for y axis, if None .label(y) is used :param errorbar_kwargs: extra dict with arguments passed to plt.errorbar :param kwargs: extra arguments passed to pylab.scatter :return: """ import pylab as plt x = _ensure_strings_from_expressions(x) y = _ensure_strings_from_expressions(y) label = str(label or selection) selection = _ensure_strings_from_expressions(selection) if length_check: count = self.count(selection=selection) if count > length_limit: raise ValueError("the number of rows (%d) is above the limit (%d), pass length_check=False, or increase length_limit" % (count, length_limit)) x_values = self.evaluate(x, selection=selection) y_values = self.evaluate(y, selection=selection) if s_expr: kwargs["s"] = self.evaluate(s_expr, selection=selection) if c_expr: kwargs["c"] = self.evaluate(c_expr, selection=selection) plt.xlabel(xlabel or self.label(x)) plt.ylabel(ylabel or self.label(y)) s = plt.scatter(x_values, y_values, label=label, **kwargs) if labels: label_values = self.evaluate(labels, selection=selection) for i, label_value in enumerate(label_values): plt.annotate(label_value, (x_values[i], y_values[i])) xerr_values = None yerr_values = None if cov is not None or corr is not None: from matplotlib.patches import Ellipse sx = self.evaluate(xerr, selection=selection) sy = self.evaluate(yerr, selection=selection) if corr is not None: sxy = self.evaluate(corr, selection=selection) * sx * sy elif cov is not None: sxy = self.evaluate(cov, selection=selection) cov_matrix = np.zeros((len(sx), 2, 2)) cov_matrix[:,0,0] = sx**2 cov_matrix[:,1,1] = sy**2 cov_matrix[:,0,1] = cov_matrix[:,1,0] = sxy ax = plt.gca() ellipse_kwargs = dict(ellipse_kwargs) ellipse_kwargs['facecolor'] = ellipse_kwargs.get('facecolor', 'none') ellipse_kwargs['edgecolor'] = ellipse_kwargs.get('edgecolor', 'black') for i in range(len(sx)): eigen_values, eigen_vectors = np.linalg.eig(cov_matrix[i]) indices = np.argsort(eigen_values)[::-1] eigen_values = eigen_values[indices] eigen_vectors = eigen_vectors[:,indices] v1 = eigen_vectors[:, 0] v2 = eigen_vectors[:, 1] varx = cov_matrix[i, 0, 0] vary = cov_matrix[i, 1, 1] angle = np.arctan2(v1[1], v1[0]) # round off errors cause negative values? if eigen_values[1] < 0 and abs((eigen_values[1]/eigen_values[0])) < 1e-10: eigen_values[1] = 0 if eigen_values[0] < 0 or eigen_values[1] < 0: raise ValueError('neg val') width, height = np.sqrt(np.max(eigen_values)), np.sqrt(np.min(eigen_values)) e = Ellipse(xy=(x_values[i], y_values[i]), width=width, height=height, angle=np.degrees(angle), **ellipse_kwargs) ax.add_artist(e) else: if xerr is not None: if _issequence(xerr): assert len(xerr) == 2, "if xerr is a sequence it should be of length 2" xerr_values = [self.evaluate(xerr[0], selection=selection), self.evaluate(xerr[1], selection=selection)] else: xerr_values = self.evaluate(xerr, selection=selection) if yerr is not None: if _issequence(yerr): assert len(yerr) == 2, "if yerr is a sequence it should be of length 2" yerr_values = [self.evaluate(yerr[0], selection=selection), self.evaluate(yerr[1], selection=selection)] else: yerr_values = self.evaluate(yerr, selection=selection) if xerr_values is not None or yerr_values is not None: errorbar_kwargs = dict(errorbar_kwargs) errorbar_kwargs['fmt'] = errorbar_kwargs.get('fmt', 'none') plt.errorbar(x_values, y_values, yerr=yerr_values, xerr=xerr_values, **errorbar_kwargs) return s
def plot1d(self, x=None, what="count(*)", grid=None, shape=64, facet=None, limits=None, figsize=None, f="identity", n=None, normalize_axis=None, xlabel=None, ylabel=None, label=None, selection=None, show=False, tight_layout=True, hardcopy=None, **kwargs): """Viz data in 1d (histograms, running means etc) Example >>> df.plot1d(df.x) >>> df.plot1d(df.x, limits=[0, 100], shape=100) >>> df.plot1d(df.x, what='mean(y)', limits=[0, 100], shape=100) If you want to do a computation yourself, pass the grid argument, but you are responsible for passing the same limits arguments: >>> counts = df.mean(df.y, binby=df.x, limits=[0, 100], shape=100)/100. >>> df.plot1d(df.x, limits=[0, 100], shape=100, grid=means, label='mean(y)/100') :param x: Expression to bin in the x direction :param what: What to plot, count(*) will show a N-d histogram, mean('x'), the mean of the x column, sum('x') the sum :param grid: If the binning is done before by yourself, you can pass it :param facet: Expression to produce facetted plots ( facet='x:0,1,12' will produce 12 plots with x in a range between 0 and 1) :param limits: list of [xmin, xmax], or a description such as 'minmax', '99%' :param figsize: (x, y) tuple passed to pylab.figure for setting the figure size :param f: transform values by: 'identity' does nothing 'log' or 'log10' will show the log of the value :param n: normalization function, currently only 'normalize' is supported, or None for no normalization :param normalize_axis: which axes to normalize on, None means normalize by the global maximum. :param normalize_axis: :param xlabel: String for label on x axis (may contain latex) :param ylabel: Same for y axis :param: tight_layout: call pylab.tight_layout or not :param kwargs: extra argument passed to pylab.plot :return: """ import pylab f = _parse_f(f) n = _parse_n(n) if type(shape) == int: shape = (shape,) binby = [] x = _ensure_strings_from_expressions(x) for expression in [x]: if expression is not None: binby = [expression] + binby limits = self.limits(binby, limits) if figsize is not None: pylab.figure(num=None, figsize=figsize, dpi=80, facecolor='w', edgecolor='k') fig = pylab.gcf() import re if facet is not None: match = re.match("(.*):(.*),(.*),(.*)", facet) if match: groups = match.groups() facet_expression = groups[0] facet_limits = [ast.literal_eval(groups[1]), ast.literal_eval(groups[2])] facet_count = ast.literal_eval(groups[3]) limits.append(facet_limits) binby.append(facet_expression) shape = (facet_count,) + shape else: raise ValueError("Could not understand 'facet' argument %r, expected something in form: 'column:-1,10:5'" % facet) if grid is None: if what: if isinstance(what, (vaex.stat.Expression)): grid = what.calculate(self, binby=binby, limits=limits, shape=shape, selection=selection) else: what = what.strip() index = what.index("(") import re groups = re.match("(.*)\((.*)\)", what).groups() if groups and len(groups) == 2: function = groups[0] arguments = groups[1].strip() functions = ["mean", "sum", "std", "count"] if function in functions: # grid = getattr(self, function)(arguments, binby, limits=limits, shape=shape, selection=selection) grid = getattr(vaex.stat, function)(arguments).calculate(self, binby=binby, limits=limits, shape=shape, selection=selection) elif function == "count" and arguments == "*": grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection) elif function == "cumulative" and arguments == "*": # TODO: comulative should also include the tails outside limits grid = self.count(binby=binby, shape=shape, limits=limits, selection=selection) grid = np.cumsum(grid) else: raise ValueError("Could not understand method: %s, expected one of %r'" % (function, functions)) else: raise ValueError("Could not understand 'what' argument %r, expected something in form: 'count(*)', 'mean(x)'" % what) else: grid = self.histogram(binby, size=shape, limits=limits, selection=selection) fgrid = f(grid) if n is not None: # ngrid = n(fgrid, axis=normalize_axis) ngrid = fgrid / fgrid.sum() else: ngrid = fgrid # reductions = [_parse_reduction(r, colormap, colors) for r in reduce] # rgrid = ngrid * 1. # for r in reduce: # r = _parse_reduction(r, colormap, colors) # rgrid = r(rgrid) # grid = self.reduce(grid, ) xmin, xmax = limits[-1] if facet: N = len(grid[-1]) else: N = len(grid) xexpression = binby[0] xar = np.arange(N + 1) / (N - 0.) * (xmax - xmin) + xmin label = str(label or selection or x) if facet: import math rows, columns = int(math.ceil(facet_count / 4.)), 4 values = np.linspace(facet_limits[0], facet_limits[1], facet_count + 1) for i in range(facet_count): ax = pylab.subplot(rows, columns, i + 1) value = ax.plot(xar, ngrid[i], drawstyle="steps-mid", label=label, **kwargs) v1, v2 = values[i], values[i + 1] pylab.xlabel(xlabel or x) pylab.ylabel(ylabel or what) ax.set_title("%3f <= %s < %3f" % (v1, facet_expression, v2)) if self.iscategory(xexpression): labels = self.category_labels(xexpression) step = len(labels) // max_labels pylab.xticks(range(len(labels))[::step], labels[::step], size='small') else: # im = pylab.imshow(rgrid, extent=np.array(limits[:2]).flatten(), origin="lower", aspect=aspect) pylab.xlabel(xlabel or self.label(x)) pylab.ylabel(ylabel or what) # print(xar, ngrid) # repeat the first element, that's how plot/steps likes it.. g = np.concatenate([ngrid[0:1], ngrid]) value = pylab.plot(xar, g, drawstyle="steps-pre", label=label, **kwargs) if self.iscategory(xexpression): labels = self.category_labels(xexpression) step = len(labels) // max_labels pylab.xticks(range(len(labels))[::step], labels[::step], size='small') if tight_layout: pylab.tight_layout() if hardcopy: pylab.savefig(hardcopy) if show: pylab.show() return value
def plot(self, x=None, y=None, z=None, what="count(*)", vwhat=None, reduce=["colormap"], f=None, normalize="normalize", normalize_axis="what", vmin=None, vmax=None, shape=256, vshape=32, limits=None, grid=None, colormap="afmhot", # colors=["red", "green", "blue"], figsize=None, xlabel=None, ylabel=None, aspect="auto", tight_layout=True, interpolation="nearest", show=False, colorbar=True, colorbar_label=None, selection=None, selection_labels=None, title=None, background_color="white", pre_blend=False, background_alpha=1., visual=dict(x="x", y="y", layer="z", fade="selection", row="subspace", column="what"), smooth_pre=None, smooth_post=None, wrap=True, wrap_columns=4, return_extra=False, hardcopy=None): """Viz data in a 2d histogram/heatmap. Declarative plotting of statistical plots using matplotlib, supports subplots, selections, layers. Instead of passing x and y, pass a list as x argument for multiple panels. Give what a list of options to have multiple panels. When both are present then will be origanized in a column/row order. This methods creates a 6 dimensional 'grid', where each dimension can map the a visual dimension. The grid dimensions are: * x: shape determined by shape, content by x argument or the first dimension of each space * y: ,, * z: related to the z argument * selection: shape equals length of selection argument * what: shape equals length of what argument * space: shape equals length of x argument if multiple values are given By default, this its shape is (1, 1, 1, 1, shape, shape) (where x is the last dimension) The visual dimensions are * x: x coordinate on a plot / image (default maps to grid's x) * y: y ,, (default maps to grid's y) * layer: each image in this dimension is blended togeher to one image (default maps to z) * fade: each image is shown faded after the next image (default mapt to selection) * row: rows of subplots (default maps to space) * columns: columns of subplot (default maps to what) All these mappings can be changes by the visual argument, some examples: >>> df.plot('x', 'y', what=['mean(x)', 'correlation(vx, vy)']) Will plot each 'what' as a column. >>> df.plot('x', 'y', selection=['FeH < -3', '(FeH >= -3) & (FeH < -2)'], visual=dict(column='selection')) Will plot each selection as a column, instead of a faded on top of each other. :param x: Expression to bin in the x direction (by default maps to x), or list of pairs, like [['x', 'y'], ['x', 'z']], if multiple pairs are given, this dimension maps to rows by default :param y: y (by default maps to y) :param z: Expression to bin in the z direction, followed by a :start,end,shape signature, like 'FeH:-3,1:5' will produce 5 layers between -10 and 10 (by default maps to layer) :param what: What to plot, count(*) will show a N-d histogram, mean('x'), the mean of the x column, sum('x') the sum, std('x') the standard deviation, correlation('vx', 'vy') the correlation coefficient. Can also be a list of values, like ['count(x)', std('vx')], (by default maps to column) :param reduce: :param f: transform values by: 'identity' does nothing 'log' or 'log10' will show the log of the value :param normalize: normalization function, currently only 'normalize' is supported :param normalize_axis: which axes to normalize on, None means normalize by the global maximum. :param vmin: instead of automatic normalization, (using normalize and normalization_axis) scale the data between vmin and vmax to [0, 1] :param vmax: see vmin :param shape: shape/size of the n-D histogram grid :param limits: list of [[xmin, xmax], [ymin, ymax]], or a description such as 'minmax', '99%' :param grid: if the binning is done before by yourself, you can pass it :param colormap: matplotlib colormap to use :param figsize: (x, y) tuple passed to pylab.figure for setting the figure size :param xlabel: :param ylabel: :param aspect: :param tight_layout: call pylab.tight_layout or not :param colorbar: plot a colorbar or not :param interpolation: interpolation for imshow, possible options are: 'nearest', 'bilinear', 'bicubic', see matplotlib for more :param return_extra: :return: """ import pylab import matplotlib n = _parse_n(normalize) if type(shape) == int: shape = (shape,) * 2 binby = [] x = _ensure_strings_from_expressions(x) y = _ensure_strings_from_expressions(y) for expression in [y, x]: if expression is not None: binby = [expression] + binby fig = pylab.gcf() if figsize is not None: fig.set_size_inches(*figsize) import re what_units = None whats = _ensure_list(what) selections = _ensure_list(selection) selections = _ensure_strings_from_expressions(selections) if y is None: waslist, [x, ] = vaex.utils.listify(x) else: waslist, [x, y] = vaex.utils.listify(x, y) x = list(zip(x, y)) limits = [limits] # every plot has its own vwhat for now vwhats = _expand_limits(vwhat, len(x)) # TODO: we're abusing this function.. logger.debug("x: %s", x) limits, shape = self.limits(x, limits, shape=shape) shape = shape[0] logger.debug("limits: %r", limits) # mapping of a grid axis to a label labels = {} shape = _expand_shape(shape, 2) vshape = _expand_shape(shape, 2) if z is not None: match = re.match("(.*):(.*),(.*),(.*)", z) if match: groups = match.groups() import ast z_expression = groups[0] logger.debug("found groups: %r", list(groups)) z_limits = [ast.literal_eval(groups[1]), ast.literal_eval(groups[2])] z_shape = ast.literal_eval(groups[3]) # for pair in x: x = [[z_expression] + list(k) for k in x] limits = np.array([[z_limits] + list(k) for k in limits]) shape = (z_shape,) + shape vshape = (z_shape,) + vshape logger.debug("x = %r", x) values = np.linspace(z_limits[0], z_limits[1], num=z_shape + 1) labels["z"] = list(["%s <= %s < %s" % (v1, z_expression, v2) for v1, v2 in zip(values[:-1], values[1:])]) else: raise ValueError("Could not understand 'z' argument %r, expected something in form: 'column:-1,10:5'" % facet) else: z_shape = 1 # z == 1 if z is None: total_grid = np.zeros((len(x), len(whats), len(selections), 1) + shape, dtype=float) total_vgrid = np.zeros((len(x), len(whats), len(selections), 1) + vshape, dtype=float) else: total_grid = np.zeros((len(x), len(whats), len(selections)) + shape, dtype=float) total_vgrid = np.zeros((len(x), len(whats), len(selections)) + vshape, dtype=float) logger.debug("shape of total grid: %r", total_grid.shape) axis = dict(plot=0, what=1, selection=2) xlimits = limits grid_axes = dict(x=-1, y=-2, z=-3, selection=-4, what=-5, subspace=-6) visual_axes = dict(x=-1, y=-2, layer=-3, fade=-4, column=-5, row=-6) # visual_default=dict(x="x", y="y", z="layer", selection="fade", subspace="row", what="column") # visual: mapping of a plot axis, to a grid axis visual_default = dict(x="x", y="y", layer="z", fade="selection", row="subspace", column="what") def invert(x): return dict((v, k) for k, v in x.items()) # visual_default_reverse = invert(visual_default) # visual_ = visual_default # visual = dict(visual) # copy for modification # add entries to avoid mapping multiple times to the same axis free_visual_axes = list(visual_default.keys()) # visual_reverse = invert(visual) logger.debug("1: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual.items(): if visual_name in free_visual_axes: free_visual_axes.remove(visual_name) else: raise ValueError("visual axes %s used multiple times" % visual_name) logger.debug("2: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual_default.items(): if visual_name in free_visual_axes and grid_name not in visual.values(): free_visual_axes.remove(visual_name) visual[visual_name] = grid_name logger.debug("3: %r %r", visual, free_visual_axes) for visual_name, grid_name in visual_default.items(): if visual_name not in free_visual_axes and grid_name not in visual.values(): visual[free_visual_axes.pop(0)] = grid_name logger.debug("4: %r %r", visual, free_visual_axes) visual_reverse = invert(visual) # TODO: the meaning of visual and visual_reverse is changed below this line, super confusing visual, visual_reverse = visual_reverse, visual # so now, visual: mapping of a grid axis to plot axis # visual_reverse: mapping of a grid axis to plot axis move = {} for grid_name, visual_name in visual.items(): if visual_axes[visual_name] in visual.values(): index = visual.values().find(visual_name) key = visual.keys()[index] raise ValueError("trying to map %s to %s while, it is already mapped by %s" % (grid_name, visual_name, key)) move[grid_axes[grid_name]] = visual_axes[visual_name] # normalize_axis = _ensure_list(normalize_axis) fs = _expand(f, total_grid.shape[grid_axes[normalize_axis]]) # assert len(vwhat) # labels["y"] = ylabels what_labels = [] if grid is None: grid_of_grids = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): grid_of_grids.append([]) for j, what in enumerate(whats): if isinstance(what, vaex.stat.Expression): grid = what.calculate(self, binby=binby, shape=shape, limits=limits, selection=selections, delay=True) else: what = what.strip() index = what.index("(") import re groups = re.match("(.*)\((.*)\)", what).groups() if groups and len(groups) == 2: function = groups[0] arguments = groups[1].strip() if "," in arguments: arguments = arguments.split(",") functions = ["mean", "sum", "std", "var", "correlation", "covar", "min", "max", "median_approx"] unit_expression = None if function in ["mean", "sum", "std", "min", "max", "median"]: unit_expression = arguments if function in ["var"]: unit_expression = "(%s) * (%s)" % (arguments, arguments) if function in ["covar"]: unit_expression = "(%s) * (%s)" % arguments if unit_expression: unit = self.unit(unit_expression) if unit: what_units = unit.to_string('latex_inline') if function in functions: grid = getattr(self, function)(arguments, binby=binby, limits=limits, shape=shape, selection=selections, delay=True) elif function == "count": grid = self.count(arguments, binby, shape=shape, limits=limits, selection=selections, delay=True) else: raise ValueError("Could not understand method: %s, expected one of %r'" % (function, functions)) else: raise ValueError("Could not understand 'what' argument %r, expected something in form: 'count(*)', 'mean(x)'" % what) if i == 0: # and j == 0: what_label = str(whats[j]) if what_units: what_label += " (%s)" % what_units if fs[j]: what_label = fs[j] + " " + what_label what_labels.append(what_label) grid_of_grids[-1].append(grid) self.executor.execute() for i, (binby, limits) in enumerate(zip(x, xlimits)): for j, what in enumerate(whats): grid = grid_of_grids[i][j].get() total_grid[i, j, :, :] = grid[:, None, ...] labels["what"] = what_labels else: dims_left = 6 - len(grid.shape) total_grid = np.broadcast_to(grid, (1,) * dims_left + grid.shape) # visual=dict(x="x", y="y", selection="fade", subspace="facet1", what="facet2",) def _selection_name(name): if name in [None, False]: return "selection: all" elif name in ["default", True]: return "selection: default" else: return "selection: %s" % name if selection_labels is None: labels["selection"] = list([_selection_name(k) for k in selections]) else: labels["selection"] = selection_labels # visual_grid = np.moveaxis(total_grid, move.keys(), move.values()) # np.moveaxis is in np 1.11 only?, use transpose axes = [None] * len(move) for key, value in move.items(): axes[value] = key visual_grid = np.transpose(total_grid, axes) logger.debug("grid shape: %r", total_grid.shape) logger.debug("visual: %r", visual.items()) logger.debug("move: %r", move) logger.debug("visual grid shape: %r", visual_grid.shape) xexpressions = [] yexpressions = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): xexpressions.append(binby[0]) yexpressions.append(binby[1]) if xlabel is None: xlabels = [] ylabels = [] for i, (binby, limits) in enumerate(zip(x, xlimits)): if z is not None: xlabels.append(self.label(binby[1])) ylabels.append(self.label(binby[2])) else: xlabels.append(self.label(binby[0])) ylabels.append(self.label(binby[1])) else: Nl = visual_grid.shape[visual_axes['row']] xlabels = _expand(xlabel, Nl) ylabels = _expand(ylabel, Nl) #labels[visual["x"]] = (xlabels, ylabels) labels["x"] = xlabels labels["y"] = ylabels # grid = total_grid # print(grid.shape) # grid = self.reduce(grid, ) axes = [] # cax = pylab.subplot(1,1,1) background_color = np.array(matplotlib.colors.colorConverter.to_rgb(background_color)) # if grid.shape[axis["selection"]] > 1:# and not facet: # rgrid = vaex.image.fade(rgrid) # finite_mask = np.any(finite_mask, axis=0) # do we really need this # print(rgrid.shape) # facet_row_axis = axis["what"] import math facet_columns = None facets = visual_grid.shape[visual_axes["row"]] * visual_grid.shape[visual_axes["column"]] if visual_grid.shape[visual_axes["column"]] == 1 and wrap: facet_columns = min(wrap_columns, visual_grid.shape[visual_axes["row"]]) wrapped = True elif visual_grid.shape[visual_axes["row"]] == 1 and wrap: facet_columns = min(wrap_columns, visual_grid.shape[visual_axes["column"]]) wrapped = True else: wrapped = False facet_columns = visual_grid.shape[visual_axes["column"]] facet_rows = int(math.ceil(facets / facet_columns)) logger.debug("facet_rows: %r", facet_rows) logger.debug("facet_columns: %r", facet_columns) # if visual_grid.shape[visual_axes["row"]] > 1: # and not wrap: # #facet_row_axis = axis["what"] # facet_columns = visual_grid.shape[visual_axes["column"]] # else: # facet_columns = min(wrap_columns, facets) # if grid.shape[axis["plot"]] > 1:# and not facet: # this loop could be done using axis arguments everywhere # assert len(normalize_axis) == 1, "currently only 1 normalization axis supported" grid = visual_grid * 1. fgrid = visual_grid * 1. ngrid = visual_grid * 1. # colorgrid = np.zeros(ngrid.shape + (4,), float) # print "norma", normalize_axis, visual_grid.shape[visual_axes[visual[normalize_axis]]] vmins = _expand(vmin, visual_grid.shape[visual_axes[visual[normalize_axis]]], type=list) vmaxs = _expand(vmax, visual_grid.shape[visual_axes[visual[normalize_axis]]], type=list) # for name in normalize_axis: visual_grid if smooth_pre: grid = vaex.grids.gf(grid, smooth_pre) if 1: axis = visual_axes[visual[normalize_axis]] for i in range(visual_grid.shape[axis]): item = [slice(None, None, None), ] * len(visual_grid.shape) item[axis] = i item = tuple(item) f = _parse_f(fs[i]) with np.errstate(divide='ignore', invalid='ignore'): # these are fine, we are ok with nan's in vaex fgrid.__setitem__(item, f(grid.__getitem__(item))) # print vmins[i], vmaxs[i] if vmins[i] is not None and vmaxs[i] is not None: nsubgrid = fgrid.__getitem__(item) * 1 nsubgrid -= vmins[i] nsubgrid /= (vmaxs[i] - vmins[i]) else: nsubgrid, vmin, vmax = n(fgrid.__getitem__(item)) vmins[i] = vmin vmaxs[i] = vmax # print " ", vmins[i], vmaxs[i] ngrid.__setitem__(item, nsubgrid) if 0: # TODO: above should be like the code below, with custom vmin and vmax grid = visual_grid[i] f = _parse_f(fs[i]) fgrid = f(grid) finite_mask = np.isfinite(grid) finite_mask = np.any(finite_mask, axis=0) if vmin is not None and vmax is not None: ngrid = fgrid * 1 ngrid -= vmin ngrid /= (vmax - vmin) ngrid = np.clip(ngrid, 0, 1) else: ngrid, vmin, vmax = n(fgrid) # vmin, vmax = np.nanmin(fgrid), np.nanmax(fgrid) # every 'what', should have its own colorbar, check if what corresponds to # rows or columns in facets, if so, do a colorbar per row or per column rows, columns = int(math.ceil(facets / float(facet_columns))), facet_columns colorbar_location = "individual" if visual["what"] == "row" and visual_grid.shape[1] == facet_columns: colorbar_location = "per_row" if visual["what"] == "column" and visual_grid.shape[0] == facet_rows: colorbar_location = "per_column" # values = np.linspace(facet_limits[0], facet_limits[1], facet_count+1) logger.debug("rows: %r, columns: %r", rows, columns) import matplotlib.gridspec as gridspec column_scale = 1 row_scale = 1 row_offset = 0 if facets > 1: if colorbar_location == "per_row": column_scale = 4 gs = gridspec.GridSpec(rows, columns * column_scale + 1) elif colorbar_location == "per_column": row_offset = 1 row_scale = 4 gs = gridspec.GridSpec(rows * row_scale + 1, columns) else: gs = gridspec.GridSpec(rows, columns) facet_index = 0 fs = _expand(f, len(whats)) colormaps = _expand(colormap, len(whats)) # row for i in range(visual_grid.shape[0]): # column for j in range(visual_grid.shape[1]): if colorbar and colorbar_location == "per_column" and i == 0: norm = matplotlib.colors.Normalize(vmins[j], vmaxs[j]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[j]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[0, j]) colorbar = fig.colorbar(sm, cax=ax, orientation="horizontal") else: colorbar = fig.colorbar(sm) if "what" in labels: label = labels["what"][j] if facets > 1: colorbar.ax.set_title(label) else: colorbar.ax.set_ylabel(colorbar_label or label) if colorbar and colorbar_location == "per_row" and j == 0: norm = matplotlib.colors.Normalize(vmins[i], vmaxs[i]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[i]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[i, -1]) colorbar = fig.colorbar(sm, cax=ax) else: colorbar = fig.colorbar(sm) label = labels["what"][i] colorbar.ax.set_ylabel(colorbar_label or label) rgrid = ngrid[i, j] * 1. # print rgrid.shape for k in range(rgrid.shape[0]): for l in range(rgrid.shape[0]): if smooth_post is not None: rgrid[k, l] = vaex.grids.gf(rgrid, smooth_post) if visual["what"] == "column": what_index = j elif visual["what"] == "row": what_index = i else: what_index = 0 if visual[normalize_axis] == "column": normalize_index = j elif visual[normalize_axis] == "row": normalize_index = i else: normalize_index = 0 for r in reduce: r = _parse_reduction(r, colormaps[what_index], []) rgrid = r(rgrid) row = facet_index // facet_columns column = facet_index % facet_columns if colorbar and colorbar_location == "individual": # visual_grid.shape[visual_axes[visual[normalize_axis]]] norm = matplotlib.colors.Normalize(vmins[normalize_index], vmaxs[normalize_index]) sm = matplotlib.cm.ScalarMappable(norm, colormaps[what_index]) sm.set_array(1) # make matplotlib happy (strange behavious) if facets > 1: ax = pylab.subplot(gs[row, column]) colorbar = fig.colorbar(sm, ax=ax) else: colorbar = fig.colorbar(sm) label = labels["what"][what_index] colorbar.ax.set_ylabel(colorbar_label or label) if facets > 1: ax = pylab.subplot(gs[row_offset + row * row_scale:row_offset + (row + 1) * row_scale, column * column_scale:(column + 1) * column_scale]) else: ax = pylab.gca() axes.append(ax) logger.debug("rgrid: %r", rgrid.shape) plot_rgrid = rgrid assert plot_rgrid.shape[1] == 1, "no layers supported yet" plot_rgrid = plot_rgrid[:, 0] if plot_rgrid.shape[0] > 1: plot_rgrid = vaex.image.fade(plot_rgrid[::-1]) else: plot_rgrid = plot_rgrid[0] extend = None if visual["subspace"] == "row": subplot_index = i elif visual["subspace"] == "column": subplot_index = j else: subplot_index = 0 extend = np.array(xlimits[subplot_index][-2:]).flatten() # extend = np.array(xlimits[i]).flatten() logger.debug("plot rgrid: %r", plot_rgrid.shape) plot_rgrid = np.transpose(plot_rgrid, (1, 0, 2)) im = ax.imshow(plot_rgrid, extent=extend.tolist(), origin="lower", aspect=aspect, interpolation=interpolation) # v1, v2 = values[i], values[i+1] def label(index, label, expression): if label and _issequence(label): return label[i] else: return self.label(expression) if visual_reverse["x"] =='x': labelsx = labels['x'] pylab.xlabel(labelsx[subplot_index]) if visual_reverse["x"] =='x': labelsy = labels['y'] pylab.ylabel(labelsy[subplot_index]) if visual["z"] in ['row']: labelsz = labels['z'] ax.set_title(labelsz[i]) if visual["z"] in ['column']: labelsz = labels['z'] ax.set_title(labelsz[j]) max_labels = 10 # xexpression = xexpressions[i] # if self.iscategory(xexpression): # labels = self.category_labels(xexpression) # step = len(labels) // max_labels # pylab.xticks(np.arange(len(labels))[::step], labels[::step], size='small') # yexpression = yexpressions[i] # if self.iscategory(yexpression): # labels = self.category_labels(yexpression) # step = len(labels) // max_labels # pylab.yticks(np.arange(len(labels))[::step], labels[::step], size='small') facet_index += 1 if title: fig.suptitle(title, fontsize="x-large") if tight_layout: if title: pylab.tight_layout(rect=[0, 0.03, 1, 0.95]) else: pylab.tight_layout() if hardcopy: pylab.savefig(hardcopy) if show: pylab.show() if return_extra: return im, grid, fgrid, ngrid, rgrid else: return im