def test_set_background(): out = tf.set_background(img1) assert out.equals(img1) sol = tf.Image(np.array([[0xff00ffff, 0xff0000ff], [0xff0000ff, 0xff00ff7d]], dtype='uint32'), coords=coords2, dims=dims) out = tf.set_background(img1, 'red') assert out.equals(sol)
def create_image(x_range=x_range, y_range=y_range, w=plot_width, h=plot_height, aggregator=ds.count(), categorical=None, black=False, cmap=None): opts={} if categorical and cmap: opts['color_key'] = categorical_color_key(len(df[aggregator.column].unique()),cmap) cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range) agg = cvs.line(df, 'longitude', 'latitude', aggregator) img = tf.shade(agg, cmap=inferno, **opts) if black: img = tf.set_background(img, 'black') return img
def export_image(img, filename, fmt=".png", _return=True, export_path=".", background=""): """Given a datashader Image object, saves it to a disk file in the requested format""" from datashader.transfer_functions import set_background if not os.path.exists(export_path): os.mkdir(export_path) if background: img=set_background(img,background) img.to_pil().save(os.path.join(export_path,filename+fmt)) return img if _return else None
def export_image(img, filename, fmt=".png", _return=True, export_path=".", background=""): """Given a datashader Image object, saves it to a disk file in the requested format""" from datashader.transfer_functions import set_background if not os.path.exists(export_path): os.mkdir(export_path) if background: img = set_background(img, background) img.to_pil().save(os.path.join(export_path, filename + fmt)) return img if _return else None
def waveforms_datashader(waveforms, threshold=None): if waveforms.shape[0]==0: return None # Make a pandas dataframe with two columns, x and y, holding all the data. The individual waveforms are separated by a row of NaNs # First downsample the waveforms 10 times (to remove the effects of 10 times upsampling during de-jittering) waveforms = waveforms[:, ::10] x_values = np.arange(len(waveforms[0])) + 1 # Then make a new array of waveforms - the last element of each waveform is a NaN new_waveforms = np.zeros((waveforms.shape[0], waveforms.shape[1] + 1)) new_waveforms[:, -1] = np.nan new_waveforms[:, :-1] = waveforms # Now make an array of x's - the last element is a NaN x = np.zeros(x_values.shape[0] + 1) x[-1] = np.nan x[:-1] = x_values # Now make the dataframe df = pd.DataFrame({'x': np.tile(x, new_waveforms.shape[0]), 'y': new_waveforms.flatten()}) # Produce a datashader canvas canvas = ds.Canvas(x_range = (np.min(x_values), np.max(x_values)), y_range = (df['y'].min() - 10, df['y'].max() + 10), plot_height=1200, plot_width=1600) # Aggregate the data agg = canvas.line(df, 'x', 'y', ds.count()) # Transfer the aggregated data to image using log transform and export the temporary image file img = tf.shade(agg, how='eq_hist') img = tf.set_background(img, 'white') # Figure sizes chosen so that the resolution is 100 dpi fig,ax = plt.subplots(1, 1, figsize = (12,8), dpi = 200) # Start plotting ax.imshow(img.to_pil()) # Set ticks/labels - 10 on each axis ax.set_xticks(np.linspace(0, 1600, 10)) ax.set_xticklabels(np.floor(np.linspace(np.min(x_values), np.max(x_values), 10))) ax.set_yticks(np.linspace(0, 1200, 10)) yticklabels = np.floor(np.linspace(df['y'].max() + 10, df['y'].min() - 10, 10)) ax.set_yticklabels(yticklabels) if threshold is not None: scaled_thresh = (threshold - np.max(yticklabels))*(1200/(np.min(yticklabels) - np.max(yticklabels))) ax.axhline(scaled_thresh, linestyle='--', color='r', alpha=0.3) # Delete the dataframe del df, waveforms, new_waveforms # Return and figure and axis for adding axis labels, title and saving the file return fig, ax
def plot_values(data_frame): canvas = ds.Canvas(plot_width=1000, plot_height=1000, x_range=(0, 1), y_range=(0, 1)) agg = canvas.points(data_frame, 'B2', 'B5') image = tf.shade(agg, cmap=['lightblue', 'darkblue'], how='linear', alpha=150) image = tf.spread(image, px=1) image = tf.set_background(image, color='#222222') return image
def plot_embedding(embedding, output_fname, plot_width=1000, plot_height=1000, cmap=fire, shade_how='eq_hist', background="black"): embedding = pd.DataFrame(data=embedding, columns=["UMAP_1", "UMAP_2"]) canvas = datashader.Canvas(plot_width=plot_width, plot_height=plot_height) canvas = canvas.points(embedding, 'UMAP_1', 'UMAP_2') canvas = datashader.transfer_functions.shade(canvas, how=shade_how, cmap=fire) if background: canvas = set_background(canvas, background) canvas.to_pil().convert('RGB').save(output_fname)
def plot_embedding_labels(embedding, output_fname, plot_width=400, plot_height=400, cmap=fire, shade_how='eq_hist', background=""): assert (embedding.shape[1] == 2) canvas = datashader.Canvas(plot_width=plot_width, plot_height=plot_height).points( embedding, 'UMAP_1', 'UMAP_2') canvas = datashader.transfer_functions.shade(canvas, how=shade_how, cmap=fire) if background: canvas = set_background(canvas, background) canvas.to_pil().convert('RGB').save(output_fname)
def draw_figure(df, fileout): geodetic = ccrs.Geodetic(globe=ccrs.Globe(datum='WGS84')) fig=plt.figure(frameon=False) tiler = StamenTerrain() ax = plt.axes(projection=tiler.crs) ax.add_feature(cartopy.feature.OCEAN,zorder=1) ax.add_feature(cartopy.feature.COASTLINE,edgecolor='green',linewidth=0.5,zorder=4) ax.add_feature(cartopy.feature.BORDERS,edgecolor='green',linewidth=0.5,zorder=4) tra = tiler.crs.transform_points(geodetic,df.lon.values,df.lat.values) x = tra[:,0] y = tra[:,1] tra = pd.DataFrame({'lon':x,'lat':y,'baroaltitude':df.baroaltitude.values}) target = 6000 ratio = target/(np.max(tra.lon)-np.min(tra.lon)) cvs = ds.Canvas(plot_width=target, plot_height=int((np.max(tra.lat)-np.min(tra.lat))*ratio)) agg = cvs.points(tra, 'lon', 'lat',ds.min('baroaltitude'))#, ds.mean('baroaltitude')) img = tf.shade(agg, cmap=inferno,how='linear') img = tf.set_background(img, 'black') r = img.to_pil() datas = r.getdata() newData = [] for item in datas: if item[0] == 0 and item[1] == 0 and item[2] == 0: newData.append((255, 255, 255, 0)) else: newData.append(item) r.putdata(newData) cax = plt.imshow(r,zorder=3,origin='upper',interpolation='gaussian',extent=(np.min(tra.lon),np.max(tra.lon),np.min(tra.lat),np.max(tra.lat))) ax1 = fig.add_axes([0.05, 0.18, 0.9, 0.025]) norm = mpl.colors.Normalize(vmin=np.min(df.baroaltitude.values)/FEET2METER, vmax=np.max(df.baroaltitude.values)/FEET2METER) cb1 = mpl.colorbar.ColorbarBase(ax1, cmap=infernompl,norm=norm,orientation='horizontal') cb1.set_label('$H_p$ [ft]') size = fig.get_size_inches() h_over_w = size[1]/size[0] fig.set_tight_layout({'pad':0}) fig.set_figwidth(TEXT_WIDTH) fig.set_figheight(TEXT_WIDTH*h_over_w) plt.savefig(fileout,format="pdf",pad_inches=0,dpi=2000, bbox_inches='tight')
def make_animation_frames(table): box = BoundingBox(Point(55.616249, 37.342988), Point(55.840990, 37.855226)) canvas = get_canvas(box, width=1000) step = 30 for hour in range(24): for period in range(60 // step): minute = period * step timestamp = time(hour, minute) view = table[(table.hour == hour) & (table.minute // step == period)] aggregate = canvas.points(view, 'pickup_longitude', 'pickup_latitude', ds.count()) aggregate = convolve_aggregate(aggregate) image = tf.shade(aggregate, cmap=viridis, how='eq_hist') image = tf.set_background(image, 'black') image = image.to_pil() # image = draw_time(image, timestamp) yield Frame(timestamp, image)
def create_image(x_range=x_range, y_range=y_range, w=plot_width, h=plot_height, aggregator=ds.count(), categorical=None, black=False, cmap=None): opts = {} if categorical and cmap: opts['color_key'] = categorical_color_key( len(df[aggregator.column].unique()), cmap) cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range) agg = cvs.line(df, 'longitude', 'latitude', aggregator) img = tf.shade(agg, cmap=inferno, **opts) if black: img = tf.set_background(img, 'black') return img
def initiate_plots(self): self.fig = plt.figure(figsize=(6, 4)) self.ax = [] self.ax.append(self.fig.add_axes([0.0, 0.3, 0.4, 1])) self.ax.append(self.fig.add_axes([0.6, 0.1, 0.3, 0.4])) self.ax.append(self.fig.add_axes([0.6, 0.6, 0.4, 0.4])) self.ax.append(self.fig.add_axes([0.0,0.0,0.2,0.2], \ facecolor ='white')) self.ax[0].hexbin(self.embedded_data[:, 0], self.embedded_data[:, 1]) self.this_spike, = self.ax[0].plot(self.embedded_data[0,0],\ self.embedded_data[0,1],'o', c='red') self.make_waveform_array() plt.sca(self.ax[2]) cvs = ds.Canvas(plot_height=400, plot_width=1000) agg = cvs.line(self.plot_waveform_frame, x='time', y = 'voltage',\ agg = ds.count()) img = tf.set_background(tf.shade(agg, how='eq_hist',cmap = 'lightblue'),\ color = (1,1,1)) img.plot() self.ylims = \ self.ax[1].set_ylim([np.min(self.waveforms),np.max(self.waveforms)]) self.ax[2].set_ylim(self.ylims) #plt.sca(self.ax[1]) #self.waveform_plot = plt.plot(self.waveforms[0,:],'x-') self.waveform_plot, = self.ax[1].plot(self.waveforms[0, :], 'x-') self.cid = \ self.fig.canvas.mpl_connect('button_press_event', self.onclick) self.radio_control = RadioButtons(self.ax[3],\ ('Show','Cluster')) self.radio_control.on_clicked(self.color_change) plt.show()
def mock_shader_func(agg, span=None): img = tf.shade(agg, cmap=viridis, span=span, how='log') img = tf.set_background(img, 'black') return img
def get_events(tdb): query = [('title', 'Prince (musician)')] for i in range(len(tdb)): events = list(tdb.trail(i, event_filter=query)) if events: yield events[0].time, events def get_dataframe(): tdb = TrailDB('pydata-tutorial.tdb') base = tdb.min_timestamp() types = [] xs = [] ys = [] #try this: #for y, (first_ts, events) in enumerate(sorted(get_events(tdb), reverse=True)): for y, (first_ts, events) in enumerate(get_events(tdb)): for event in events: xs.append(int(event.time - base) / (24 * 3600)) ys.append(y) types.append('user' if event.user else 'anon') data = pd.DataFrame({'x': xs, 'y': ys}) data['type'] = pd.Series(types, dtype='category') return data cnv = ds.Canvas(400, 300) agg = cnv.points(get_dataframe(), 'x', 'y', ds.count_cat('type')) colors = {'anon': 'red', 'user': '******'} img=tf.set_background(tf.colorize(agg, colors, how='eq_hist'), 'white') with open('prince.png', 'w') as f: f.write(img.to_bytesio().getvalue())
def datashade_points(points, ax=None, labels=None, values=None, cmap="Blues", color_key=None, color_key_cmap="Spectral", background="white", vrange=None, width=800, height=800, show_legend=True): if ax is None: dpi = plt.rcParams["figure.dpi"] fig = plt.figure(figsize=(width / dpi, height / dpi)) ax = fig.add_subplot(111) extent = _get_extent(points) canvas = ds.Canvas( plot_width=width, plot_height=height, x_range=(extent[0], extent[1]), y_range=(extent[2], extent[3]), ) data = pd.DataFrame(points, columns=("x", "y")) legend_elements = None # Color by labels if labels is not None: if labels.shape[0] != points.shape[0]: raise ValueError("Labels must have a label for " "each sample (size mismatch: {} {})".format( labels.shape[0], points.shape[0])) data["label"] = pd.Categorical(labels) aggregation = canvas.points(data, "x", "y", agg=ds.count_cat("label")) if color_key is None and color_key_cmap is None: result = tf.shade(aggregation, how="eq_hist") elif color_key is None: unique_labels = np.unique(labels) num_labels = unique_labels.shape[0] color_key = _to_hex( plt.get_cmap(color_key_cmap)(np.linspace(0, 1, num_labels))) legend_elements = [ Patch(facecolor=color_key[i], label=k) for i, k in enumerate(unique_labels) ] result = tf.shade(aggregation, color_key=color_key, how="eq_hist") else: legend_elements = [ Patch(facecolor=color_key[k], label=k) for k in color_key.keys() ] result = tf.shade(aggregation, color_key=color_key, how="eq_hist") # Color by values elif values is not None: if values.shape[0] != points.shape[0]: raise ValueError("Values must have a value for " "each sample (size mismatch: {} {})".format( values.shape[0], points.shape[0])) unique_values = np.unique(values) if unique_values.shape[0] >= 256: if vrange is None: min_val, max_val = np.min(values), np.max(values) bin_size = (max_val - min_val) / 255.0 data["val_cat"] = pd.Categorical( np.round((values - min_val) / bin_size).astype(np.int32)) else: (min_val, max_val) = vrange bin_size = (max_val - min_val) / 255.0 scaled = np.round( (values - min_val) / bin_size).astype(np.int32) scaled[scaled < 0] = 0 scaled[scaled > 255] = 255 data["val_cat"] = pd.Categorical(scaled) aggregation = canvas.points(data, "x", "y", agg=ds.count_cat("val_cat")) color_key = _to_hex(plt.get_cmap(cmap)(np.linspace(0, 1, 256))) result = tf.shade(aggregation, color_key=color_key, how="eq_hist") else: data["val_cat"] = pd.Categorical(values) aggregation = canvas.points(data, "x", "y", agg=ds.count_cat("val_cat")) color_key_cols = _to_hex( plt.get_cmap(cmap)(np.linspace(0, 1, unique_values.shape[0]))) color_key = dict(zip(unique_values, color_key_cols)) result = tf.shade(aggregation, color_key=color_key, how="eq_hist") # Color by density (default datashader option) else: aggregation = canvas.points(data, "x", "y", agg=ds.count()) result = tf.shade(aggregation, cmap=plt.get_cmap(cmap)) if background is not None: result = tf.set_background(result, background) _embed_datashader_in_an_axis(result, ax) if show_legend and legend_elements is not None: ax.legend(handles=legend_elements) ax.set(xticks=[], yticks=[]) plt.show() return ax
def show_velphase(ds, ray_df, ray_start, ray_end, triray, filename): # take in the yt dataset (ds) and a ray as a dataframe # preliminaries rs = ray_start.ndarray_view() re = ray_end.ndarray_view() imsize = 500 core_width = 10. proper_box_size = ds.get_parameter( 'CosmologyComovingBoxSize') / ds.get_parameter( 'CosmologyHubbleConstantNow') * 1000. # in kpc redshift = ds.get_parameter('CosmologyCurrentRedshift') # take out a "core sample" that extends along the ray with a width given by core_width ad = ds.r[rs[0]:re[0], rs[1] - 0.5 * core_width / proper_box_size:rs[1] + 0.5 * core_width / proper_box_size, rs[2] - 0.5 * core_width / proper_box_size:rs[2] + 0.5 * core_width / proper_box_size] cell_vol = ad["cell_volume"] cell_mass = ad["cell_mass"] cell_size = np.array(cell_vol)**(1. / 3.) * proper_box_size x_cells = ad['x'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) y_cells = ad['y'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) z_cells = ad['z'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) dens = np.log10(ad['density'].ndarray_view()) temp = np.log10(ad['temperature'].ndarray_view()) phase = np.chararray(np.size(temp), 4) phase[temp < 19.] = 'hot' phase[temp < 6.] = 'warm' phase[temp < 5.] = 'cool' phase[temp < 4.] = 'cold' df = pd.DataFrame({ 'x': x_cells, 'y': y_cells, 'z': z_cells, 'vx': ad["x-velocity"], 'vy': ad["y-velocity"], 'vz': ad["z-velocity"], 'temp': temp, 'dens': dens, 'phase': phase }) df.phase = df.phase.astype('category') cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(np.min(df['x']), np.max(df['x'])), y_range=(np.mean(df['y']) - 100. / 0.695, np.mean(df['y']) + 100. / 0.695)) agg = cvs.points(df, 'x', 'y', dshader.count_cat('phase')) img = tf.shade(agg, color_key=phase_color_key) x_y = tf.spread(img, px=2, shape='square') x_y.to_pil().save(filename + '_x_vs_y.png') cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(np.min(df['x']), np.max(df['x'])), y_range=(-0.008, 0.008)) agg = cvs.points(df, 'x', 'vx', dshader.count_cat('phase')) img = tf.shade(agg, color_key=phase_color_key) x_vx = tf.spread(img, px=1, shape='square') x_vx.to_pil().save(filename + '_x_vs_vx.png') for species in species_dict.keys(): cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(rs[0], re[0]), y_range=(-0.008, 0.008)) vx = tf.shade(cvs.points(ray_df, 'x', 'x-velocity', agg=reductions.mean(species_dict[species])), how='eq_hist') tf.set_background(vx, "white") ray_vx = tf.spread(vx, px=4, shape='square') pil = ray_vx.to_pil() pil.save(filename + '_' + species + '_ray_vx.png', format='png')
shader_data = pd.DataFrame( np.hstack([transformed, loda_scores[:, np.newaxis]]), columns=["x", "y", "anomaly_score"], ) agg = ds.Canvas(plot_width=1000, plot_height=800).points(shader_data, "x", "y", ds.mean("anomaly_score")) img = tf.shade( agg, cmap=fire, name="Transformation by UMAP + Datashader (average anomaly score)") img = tf.set_background(img, "black") plt.figure(figsize=(15, 15)) plt.subplot(111, aspect="auto") plt.subplots_adjust(left=0.02, right=0.98, bottom=0.001, top=0.96, wspace=0.05, hspace=0.01) plt.imshow(img.to_pil()) plt.title("Transformation by UMAP + Datashader (average anomaly score)", fontsize=15) plt.xticks(()) plt.yticks(()) plt.show()
def connectivity( umap_object, edge_bundling=None, edge_cmap="gray_r", show_points=False, labels=None, values=None, theme=None, cmap="Blues", color_key=None, color_key_cmap="Spectral", background="white", width=800, height=800, ): """Plot connectivity relationships of the underlying UMAP simplicial set data structure. Internally UMAP will make use of what can be viewed as a weighted graph. This graph can be plotted using the layout provided by UMAP as a potential diagnostic view of the embedding. Currently this only works for 2D embeddings. While there are many optional parameters to further control and tailor the plotting, you need only pass in the trained/fit umap model to get results. This plot utility will attempt to do the hard work of avoiding overplotting issues and provide options for plotting the points as well as using edge bundling for graph visualization. Parameters ---------- umap_object: trained UMAP object A trained UMAP object that has a 2D embedding. edge_bundling: string or None (optional, default None) The edge bundling method to use. Currently supported are None or 'hammer'. See the datashader docs on graph visualization for more details. edge_cmap: string (default 'gray_r') The name of a matplotlib colormap to use for shading/ coloring the edges of the connectivity graph. Note that the ``theme``, if specified, will override this. show_points: bool (optional False) Whether to display the points over top of the edge connectivity. Further options allow for coloring/ shading the points accordingly. labels: array, shape (n_samples,) (optional, default None) An array of labels (assumed integer or categorical), one for each data sample. This will be used for coloring the points in the plot according to their label. Note that this option is mutually exclusive to the ``values`` option. values: array, shape (n_samples,) (optional, default None) An array of values (assumed float or continuous), one for each sample. This will be used for coloring the points in the plot according to a colorscale associated to the total range of values. Note that this option is mutually exclusive to the ``labels`` option. theme: string (optional, default None) A color theme to use for plotting. A small set of predefined themes are provided which have relatively good aesthetics. Available themes are: * 'blue' * 'red' * 'green' * 'inferno' * 'fire' * 'viridis' * 'darkblue' * 'darkred' * 'darkgreen' cmap: string (optional, default 'Blues') The name of a matplotlib colormap to use for coloring or shading points. If no labels or values are passed this will be used for shading points according to density (largely only of relevance for very large datasets). If values are passed this will be used for shading according the value. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. color_key: dict or array, shape (n_categories) (optional, default None) A way to assign colors to categoricals. This can either be an explicit dict mapping labels to colors (as strings of form '#RRGGBB'), or an array like object providing one color for each distinct category being provided in ``labels``. Either way this mapping will be used to color points according to the label. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. color_key_cmap: string (optional, default 'Spectral') The name of a matplotlib colormap to use for categorical coloring. If an explicit ``color_key`` is not given a color mapping for categories can be generated from the label list and selecting a matching list of colors from the given colormap. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. background: string (optional, default 'white) The color of the background. Usually this will be either 'white' or 'black', but any color name will work. Ideally one wants to match this appropriately to the colors being used for points etc. This is one of the things that themes handle for you. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. width: int (optional, default 800) The desired width of the plot in pixels. height: int (optional, default 800) The desired height of the plot in pixels Returns ------- result: matplotlib axis The result is a matplotlib axis with the relevant plot displayed. If you are using a notbooks and have ``%matplotlib inline`` set then this will simply display inline. """ if theme is not None: cmap = _themes[theme]["cmap"] color_key_cmap = _themes[theme]["color_key_cmap"] edge_cmap = _themes[theme]["edge_cmap"] background = _themes[theme]["background"] points = umap_object.embedding_ point_df = pd.DataFrame(points, columns=("x", "y")) point_size = 100.0 / np.sqrt(points.shape[0]) if point_size > 1: px_size = int(np.round(point_size)) else: px_size = 1 if show_points: edge_how = "log" else: edge_how = "eq_hist" coo_graph = umap_object.graph_.tocoo() edge_df = pd.DataFrame( np.vstack([coo_graph.row, coo_graph.col, coo_graph.data]).T, columns=("source", "target", "weight"), ) edge_df["source"] = edge_df.source.astype(np.int32) edge_df["target"] = edge_df.target.astype(np.int32) extent = _get_extent(points) canvas = ds.Canvas( plot_width=width, plot_height=height, x_range=(extent[0], extent[1]), y_range=(extent[2], extent[3]), ) if edge_bundling is None: edges = bd.directly_connect_edges(point_df, edge_df, weight="weight") elif edge_bundling == "hammer": warn("Hammer edge bundling is expensive for large graphs!\n" "This may take a long time to compute!") edges = bd.hammer_bundle(point_df, edge_df, weight="weight") else: raise ValueError( "{} is not a recognised bundling method".format(edge_bundling)) edge_img = tf.shade( canvas.line(edges, "x", "y", agg=ds.sum("weight")), cmap=plt.get_cmap(edge_cmap), how=edge_how, ) edge_img = tf.set_background(edge_img, background) if show_points: point_img = _datashade_points( points, None, labels, values, cmap, color_key, color_key_cmap, None, width, height, False, ) if px_size > 1: point_img = tf.dynspread(point_img, threshold=0.5, max_px=px_size) result = tf.stack(edge_img, point_img, how="over") else: result = edge_img font_color = _select_font_color(background) dpi = plt.rcParams["figure.dpi"] fig = plt.figure(figsize=(width / dpi, height / dpi)) ax = fig.add_subplot(111) _embed_datashader_in_an_axis(result, ax) ax.set(xticks=[], yticks=[]) ax.text( 0.99, 0.01, "UMAP: n_neighbors={}, min_dist={}".format(umap_object.n_neighbors, umap_object.min_dist), transform=ax.transAxes, horizontalalignment="right", color=font_color, ) return ax
for i in range(len(tdb)): events = list(tdb.trail(i, event_filter=query)) if events: yield events[0].time, events def get_dataframe(): tdb = TrailDB('pydata-tutorial.tdb') base = tdb.min_timestamp() types = [] xs = [] ys = [] # try this: # for y, (first_ts, events) in enumerate(sorted(get_events(tdb), reverse=True)): for y, (first_ts, events) in enumerate(get_events(tdb)): for event in events: xs.append(old_div(int(event.time - base), (24 * 3600))) ys.append(y) types.append('user' if event.user else 'anon') data = pd.DataFrame({'x': xs, 'y': ys}) data['type'] = pd.Series(types, dtype='category') return data cnv = ds.Canvas(400, 300) agg = cnv.points(get_dataframe(), 'x', 'y', ds.count_cat('type')) colors = {'anon': 'red', 'user': '******'} img = tf.set_background(tf.colorize(agg, colors, how='eq_hist'), 'white') with open('prince.png', 'w') as f: f.write(img.to_bytesio().getvalue())
from datashader import utils os.chdir("//sbs2003/Daten-CME/") t1 = time.time() def data_pool(file): df = dd.read_parquet(file) print(file + " loaded") return df data = None if __name__ == '__main__': print(datetime.datetime.now()) t1 = time.time() files = glob.iglob('*.csv_2_.parquet') p = Pool(os.cpu_count()) data = dd.concat(p.map(data_pool, files)) # reset_index(drop=True)) canvas = ds.Canvas(x_range=(-74.25, -73.7), y_range=(40.5, 41), plot_width=8000, plot_height=8000) agg = canvas.points(data, 'End_Lon', 'End_Lat') pic = tf.set_background(tf.shade(agg, cmap=reversed(blues)), color="#364564") #364564 utils.export_image(pic, "NYCPlot fn1", fmt=".png") print("time needed", time.time() - t1)
def bg(img): return tf.set_background(img,"black")
def connectivity_base( x: int, y: int, edge_df: pd.DataFrame, highlights: Optional[list] = None, edge_bundling: Optional[str] = None, edge_cmap: str = "gray_r", show_points: bool = True, labels: Optional[list] = None, values: Optional[list] = None, theme: Optional[str] = None, cmap: str = "Blues", color_key: Union[dict, list, None] = None, color_key_cmap: str = "Spectral", background: str = "black", figsize: tuple = (7, 5), ax: Optional[Axes] = None, sort: str = "raw", save_show_or_return: str = "return", save_kwargs: dict = {}, ) -> Union[None, Axes]: """Plot connectivity relationships of the underlying UMAP simplicial set data structure. Internally UMAP will make use of what can be viewed as a weighted graph. This graph can be plotted using the layout provided by UMAP as a potential diagnostic view of the embedding. Currently this only works for 2D embeddings. While there are many optional parameters to further control and tailor the plotting, you need only pass in the trained/fit umap model to get results. This plot utility will attempt to do the hard work of avoiding overplotting issues and provide options for plotting the points as well as using edge bundling for graph visualization. Parameters ---------- x: `int` The first component of the embedding. y: `int` The second component of the embedding. edge_df `pd.DataFrame` The dataframe denotes the graph edge pairs. The three columns include 'source', 'target' and 'weight'. highlights: `list`, `list of list` or None (default: `None`) The list that cells will be restricted to. edge_bundling: string or None (optional, default None) The edge bundling method to use. Currently supported are None or 'hammer'. See the datashader docs on graph visualization for more details. edge_cmap: string (default 'gray_r') The name of a matplotlib colormap to use for shading/ coloring the edges of the connectivity graph. Note that the ``theme``, if specified, will override this. show_points: bool (optional False) Whether to display the points over top of the edge connectivity. Further options allow for coloring/ shading the points accordingly. labels: array, shape (n_samples,) (optional, default None) An array of labels (assumed integer or categorical), one for each data sample. This will be used for coloring the points in the plot according to their label. Note that this option is mutually exclusive to the ``values`` option. values: array, shape (n_samples,) (optional, default None) An array of values (assumed float or continuous), one for each sample. This will be used for coloring the points in the plot according to a colorscale associated to the total range of values. Note that this option is mutually exclusive to the ``labels`` option. theme: string (optional, default None) A color theme to use for plotting. A small set of predefined themes are provided which have relatively good aesthetics. Available themes are: * 'blue' * 'red' * 'green' * 'inferno' * 'fire' * 'viridis' * 'darkblue' * 'darkred' * 'darkgreen' cmap: string (optional, default 'Blues') The name of a matplotlib colormap to use for coloring or shading points. If no labels or values are passed this will be used for shading points according to density (largely only of relevance for very large datasets). If values are passed this will be used for shading according the value. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. color_key: dict or array, shape (n_categories) (optional, default None) A way to assign colors to categoricals. This can either be an explicit dict mapping labels to colors (as strings of form '#RRGGBB'), or an array like object providing one color for each distinct category being provided in ``labels``. Either way this mapping will be used to color points according to the label. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. color_key_cmap: string (optional, default 'Spectral') The name of a matplotlib colormap to use for categorical coloring. If an explicit ``color_key`` is not given a color mapping for categories can be generated from the label list and selecting a matching list of colors from the given colormap. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. background: string (optional, default 'white) The color of the background. Usually this will be either 'white' or 'black', but any color name will work. Ideally one wants to match this appropriately to the colors being used for points etc. This is one of the things that themes handle for you. Note that if theme is passed then this value will be overridden by the corresponding option of the theme. width: int (optional, default 800) The desired width of the plot in pixels. height: int (optional, default 800) The desired height of the plot in pixels sort: `str` (optional, default `raw`) The method to reorder data so that high values points will be on top of background points. Can be one of {'raw', 'abs'}, i.e. sorted by raw data or sort by absolute values. save_show_or_return: {'show', 'save', 'return'} (default: `return`) Whether to save, show or return the figure. save_kwargs: `dict` (default: `{}`) A dictionary that will passed to the save_fig function. By default it is an empty dictionary and the save_fig function will use the {"path": None, "prefix": 'connectivity_base', "dpi": None, "ext": 'pdf', "transparent": True, "close": True, "verbose": True} as its parameters. Otherwise you can provide a dictionary that properly modify those keys according to your needs. Returns ------- result: Either return None or a matplotlib axis with the relevant plot displayed based on arguments. If you are using a notbooks and have ``%matplotlib inline`` set then this will simply display inline. """ import matplotlib.pyplot as plt import datashader as ds import datashader.transfer_functions as tf import datashader.bundling as bd dpi = plt.rcParams["figure.dpi"] if theme is not None: cmap = _themes[theme]["cmap"] color_key_cmap = _themes[theme]["color_key_cmap"] edge_cmap = _themes[theme]["edge_cmap"] background = _themes[theme]["background"] points = np.array([x, y]).T point_df = pd.DataFrame(points, columns=("x", "y")) point_size = 500.0 / np.sqrt(points.shape[0]) if point_size > 1: px_size = int(np.round(point_size)) else: px_size = 1 if show_points: edge_how = "log" else: edge_how = "eq_hist" extent = _get_extent(points) canvas = ds.Canvas( plot_width=int(figsize[0] * dpi), plot_height=int(figsize[1] * dpi), x_range=(extent[0], extent[1]), y_range=(extent[2], extent[3]), ) if edge_bundling is None: edges = bd.directly_connect_edges(point_df, edge_df, weight="weight") elif edge_bundling == "hammer": warn("Hammer edge bundling is expensive for large graphs!\n" "This may take a long time to compute!") edges = bd.hammer_bundle(point_df, edge_df, weight="weight") else: raise ValueError( "{} is not a recognised bundling method".format(edge_bundling)) edge_img = tf.shade( canvas.line(edges, "x", "y", agg=ds.sum("weight")), cmap=plt.get_cmap(edge_cmap), how=edge_how, ) edge_img = tf.set_background(edge_img, background) if show_points: point_img = _datashade_points( points, None, labels, values, highlights, cmap, color_key, color_key_cmap, None, figsize[0] * dpi, figsize[1] * dpi, True, sort=sort, ) if px_size > 1: point_img = tf.dynspread(point_img, threshold=0.5, max_px=px_size) result = tf.stack(edge_img, point_img, how="over") else: result = edge_img if ax is None: fig = plt.figure(figsize=figsize) ax = fig.add_subplot(111) _embed_datashader_in_an_axis(result, ax) ax.set(xticks=[], yticks=[]) if save_show_or_return == "save": s_kwargs = { "path": None, "prefix": "connectivity_base", "dpi": None, "ext": "pdf", "transparent": True, "close": True, "verbose": True, } s_kwargs = update_dict(s_kwargs, save_kwargs) save_fig(**s_kwargs) elif save_show_or_return == "show": plt.tight_layout() plt.show() elif save_show_or_return == "return": return ax
#sparse.save_npz('1e6_factors_mat.npz', mat_csr) reducer = umap.UMAP(metric='cosine', verbose=2, n_epochs=100) # takes about 2 hours with an i7 7700 ;_;7 embedding = reducer.fit_transform(mat_csr) np.save('embedding_primes', embedding) mat = np.load('embedding_primes.npy') df = pd.DataFrame(mat, columns=['x', 'y']) cvs = ds.Canvas(plot_width=500, plot_height=500) agg = cvs.points(df, 'x', 'y') img = tf.shade(agg, how='eq_hist', cmap=mp.cm.viridis) tf.set_background(img, 'black') fig = plt.figure(figsize=(10, 10)) fig.patch.set_facecolor('black') plt.scatter(df.x, df.y, marker='o', s=1, edgecolor='', c=df.index, cmap="magma", alpha=0.5) plt.axis("off") plt.show()