def test_dynspread(): b = 0xffff0000 data = np.array([[b, b, 0, 0, 0], [b, b, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, b, 0], [0, 0, 0, 0, 0]], dtype='uint32') coords = [np.arange(5), np.arange(5)] img = tf.Image(data, coords=coords, dims=dims) assert tf.dynspread(img).equals(tf.spread(img, 1)) assert tf.dynspread(img, threshold=0.9).equals(tf.spread(img, 2)) assert tf.dynspread(img, threshold=0).equals(img) assert tf.dynspread(img, max_px=0).equals(img) pytest.raises(ValueError, lambda: tf.dynspread(img, threshold=1.1)) pytest.raises(ValueError, lambda: tf.dynspread(img, max_px=-1))
def render_image(self): pix = tf.shade(self.agg, cmap=self.color_ramp, color_key=self.colormap, how=self.transfer_function) if self.spread_size > 0: pix = tf.spread(pix, px=self.spread_size) return pix
def test_categorical_dynspread(): a_data = np.array([[1, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]], dtype='int32') b_data = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]], dtype='int32') c_data = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]], dtype='int32') data = np.dstack([a_data, b_data, c_data]) coords = [np.arange(5), np.arange(5)] arr = xr.DataArray(data, coords=coords + [['a', 'b', 'c']], dims=dims + ['cat']) assert tf.dynspread(arr, threshold=0.4).equals(tf.spread(arr, 0)) assert tf.dynspread(arr, threshold=0.7).equals(tf.spread(arr, 1)) assert tf.dynspread(arr, threshold=1.0).equals(tf.spread(arr, 3)) assert tf.dynspread(arr, max_px=0).equals(arr)
def viewInteractiveImage(x_range, y_range, w, h, data_source, **kwargs): dd = data_source[[self.x, self.y, self.aggregate_col]] dd[self.x] = self._to_xaxis_type(dd[self.x]) dd[self.y] = self._to_yaxis_type(dd[self.y]) x_range = self._to_xaxis_type(x_range) y_range = self._to_yaxis_type(y_range) cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range) aggregator, cmap = _compute_datashader_assets( dd, self.x, self.aggregate_col, self.aggregate_fn, self.color_palette, ) agg = cvs.points( dd, self.x, self.y, aggregator, ) if self.constant_limit is None or self.aggregate_fn == "count": self.constant_limit = [ float(cp.nanmin(agg.data)), float(cp.nanmax(agg.data)), ] self.render_legend() span = {"span": self.constant_limit} if self.pixel_shade_type == "eq_hist": span = {} img = tf.shade(agg, how=self.pixel_shade_type, **cmap, **span) if self.pixel_spread == "dynspread": return tf.dynspread( img, threshold=self.pixel_density, max_px=self.point_size, shape=self.point_shape, ) else: return tf.spread(img, px=self.point_size, shape=self.point_shape)
def render_image(self): if self.colormaps: colormap = self.colormaps.get(self.field_title, None) if colormap: self.colormap = colormap self.colornames = self.color_name_maps[self.field_title] pix = tf.shade(self.agg, cmap=self.color_ramp, color_key=self.colormap, how=self.transfer_function) if self.spread_size > 0: pix = tf.spread(pix, px=self.spread_size) return pix
def plot_values(data_frame): canvas = ds.Canvas(plot_width=1000, plot_height=1000, x_range=(0, 1), y_range=(0, 1)) agg = canvas.points(data_frame, 'B2', 'B5') image = tf.shade(agg, cmap=['lightblue', 'darkblue'], how='linear', alpha=150) image = tf.spread(image, px=1) image = tf.set_background(image, color='#222222') return image
def render_image(self): # handle categorical field if self.field in self.categorical_fields: pix = tf.colorize(self.agg, self.colormap, how=self.transfer_function) # handle ordinal field elif self.field in self.ordinal_fields: pix = tf.interpolate(self.agg, cmap=self.color_ramp, how=self.transfer_function) # handle no field else: pix = tf.interpolate(self.agg, cmap=self.color_ramp, how=self.transfer_function) if self.spread_size > 0: pix = tf.spread(pix, px=self.spread_size) return pix
def create_datashader_tile(gdf, x, y, zoom): tile = mercantile.Tile(x, y, zoom) bounds = mercantile.bounds(tile) geom = shapely.geometry.box(*bounds) mask = gdf.within(geom) xy_bounds = mercantile.xy_bounds(tile) cvs = ds.Canvas(plot_width=256, plot_height=256, x_range=(xy_bounds[0], xy_bounds[2]), y_range=(xy_bounds[1], xy_bounds[3])) agg = cvs.points(gdf[mask], 'x', 'y', agg=ds.count()) img = tf.shade(agg, cmap=['blue', 'darkblue', 'black'], how='eq_hist') img = tf.spread(img, name="spread 1px") return img.to_pil()
def master_tile(data, img, opts, z): """ makes the map at zoom level = z to be broken down into tiles :param zoom_level: :return: """ dim = map_size(z) proj_data = proj_factor(z, img) * data proj_data.y = dim - proj_data.y scene = ds.Canvas(x_range=[0, dim], y_range=[0, dim], plot_width=dim, plot_height=dim) aggregation = scene.points(proj_data, 'x', 'y') alpha = opts['dot_alpha'](z) image = tf.shade(aggregation, cmap=opts['dot_color'], alpha=alpha) if opts['dot_size']: image = tf.spread(image, px=opts['dot_size'], shape='circle', name="spread square") export_image(image, 'master_tile', background=None) return image
def serve_image(): #dataset # parse params print('-------------- server receives request-----------') width = int(request.args.get('width')) height = int(request.args.get('height')) spread = int(request.args.get('spread')) xmin = float(request.args.get('xmin')) xmax = float(request.args.get('xmax')) ymax = float(request.args.get('ymax')) ymin = float(request.args.get('ymin')) colorw = str(request.args.get('colorw')) colorb = str(request.args.get('colorb')) colora = str(request.args.get('colora')) colorh = str(request.args.get('colorh')) coloro = str(request.args.get('coloro')) x_range = webm(latitude=ymin, longitude=xmin) y_range = webm(latitude=ymax, longitude=xmax) cvs = ds.Canvas(plot_width=width, plot_height=height, x_range=(x_range[0], y_range[0]), y_range=(x_range[1], y_range[1])) agg = cvs.points(df, 'easting', 'northing', ds.count_cat('race')) color_key = { 'w': colorw, 'b': colorb, 'a': colora, 'h': colorh, 'o': coloro } img = tf.shade(agg, color_key=color_key, how='eq_hist') img = tf.spread(img, px=spread) img_io = img.to_bytesio() return send_file(img_io, mimetype='image/png')
def viewInteractiveImage(x_range, y_range, w, h, data_source): cvs = cds.Canvas( plot_width=w, plot_height=h, x_range=x_range, y_range=y_range ) agg = cvs.points( data_source, self.x, self.y, getattr(cds, self.aggregate_fn)(self.aggregate_col), ) img = tf.shade( agg, cmap=self.color_palette, how=self.pixel_shade_type ) if self.pixel_spread == "dynspread": return tf.dynspread( img, threshold=self.pixel_density, max_px=self.point_size, shape=self.point_shape, ) else: return tf.spread( img, px=self.point_size, shape=self.point_shape )
def test_spread(): p = 0x7d00007d g = 0x7d00FF00 b = 0x7dFF0000 data = np.array([[p, p, 0, 0, 0], [p, g, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, b, 0], [0, 0, 0, 0, 0]], dtype='uint32') coords = [np.arange(5), np.arange(5)] img = tf.Image(data, coords=coords, dims=dims) s = tf.spread(img) o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000], [0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000], [0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]]) np.testing.assert_equal(s.data, o) assert (s.x_axis == img.x_axis).all() assert (s.y_axis == img.y_axis).all() assert s.dims == img.dims s = tf.spread(img, px=2) o = np.array([[0xed00863b, 0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000], [0xed00863b, 0xed00863b, 0xf581411c, 0xdc904812, 0x7dff0000], [0xed00863b, 0xf581411c, 0xed864419, 0xbca85600, 0x7dff0000], [0xbc00a82a, 0xdc904812, 0xbca85600, 0x7dff0000, 0x7dff0000], [0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000, 0x7dff0000]]) np.testing.assert_equal(s.data, o) s = tf.spread(img, shape='square') o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000], [0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000], [0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]]) np.testing.assert_equal(s.data, o) s = tf.spread(img, how='add') o = np.array([[0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000], [0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000], [0xfa007f3e, 0xfa007f3e, 0xfa7f7f00, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000], [0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]]) np.testing.assert_equal(s.data, o) mask = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]]) s = tf.spread(img, mask=mask) o = np.array([[0xbc00a82a, 0xbc00007d, 0x7d00ff00, 0x00000000, 0x00000000], [0xbc00007d, 0xbc00a82a, 0x7d00007d, 0x00000000, 0x00000000], [0x7d00ff00, 0x7d00007d, 0xbca85600, 0x00000000, 0x7dff0000], [0x00000000, 0x00000000, 0x00000000, 0x7dff0000, 0x00000000], [0x00000000, 0x00000000, 0x7dff0000, 0x00000000, 0x7dff0000]]) np.testing.assert_equal(s.data, o) s = tf.spread(img, px=0) np.testing.assert_equal(s.data, img.data) pytest.raises(ValueError, lambda: tf.spread(img, px=-1)) pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones(2))) pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones((2, 2))))
def my_nodesplot(nodes, name=None, canvas=None, cat=None): canvas = ds.Canvas(**cvsopts) if canvas is None else canvas aggregator=None if cat is None else ds.count_cat(cat) agg=canvas.points(nodes,'x','y',aggregator) return tf.spread(tf.shade(agg, cmap=["#333333"], color_key=colors, min_alpha=255), px=3, name=name)
def _apply_spreading(self, array): img = tf.Image(array) return tf.spread(img, px=self.p.px, how=self.p.how, shape=self.p.shape).data
def main(args): parser = argparse.ArgumentParser('Clustering with KlustaKwik') parser.add_argument( 'target', help= 'Target path, either path containing tetrode files, or single tetrodeXX.fet.0' ) parser.add_argument('--KK', help='Path to KlustaKwik executable') parser.add_argument('--features', nargs='*', help='list of features to use for clustering') parser.add_argument('--config', help='Path to configuration file') parser.add_argument('--cluster', action='store_true', help='Directly run ') parser.add_argument('--force', help='Overwrite existing files.', action='store_true') parser.add_argument('--skip', help='Skip if clu file exists already', action='store_true') parser.add_argument('--no_spread', help='Shade report plots without static spread', action='store_true') cli_args = parser.parse_args(args) # Load default configuration yaml file default_cfg_path = Path( pkg_resources.resource_filename( __name__, '../resources/cluster_defaults.yml')).resolve() if not default_cfg_path.exists(): logging.error('Could not find default config file.') raise FileNotFoundError logger.debug('Loading default configuration') cfg = load_yaml(default_cfg_path) # Load local config file if it exists local_cfg_path = Path( pkg_resources.resource_filename( __name__, '../resources/cluster_defaults_local.yml')).resolve() if local_cfg_path.exists(): logger.debug('Loading and updating with local configuration') local_cfg = load_yaml(local_cfg_path) cfg.update(local_cfg) # Load custom config path custom_cfg_path = Path( cli_args.config).resolve() if cli_args.config else None if custom_cfg_path: if custom_cfg_path.exists(): logger.debug('Loading and updating with custom configuration') cfg.update(load_yaml(custom_cfg_path)) else: raise FileNotFoundError( f"Could not load configuration file {custom_cfg_path}") # Load parameters from command line logger.debug('Parsing and updating configuration with CLI arguments') cfg.update(vars(cli_args)) logger.debug(cfg) # try to find Klustakwik executable if necessary... if cli_args.KK is None: cli_args.KK = shutil.which('KlustaKwik') or shutil.which( 'klustakwik') or shutil.which('Klustakwik') if cli_args.KK is None and cli_args.cluster: raise FileNotFoundError( 'Could not find the KlustaKwik executable on the path, and none given.' ) # # Building KlustaKwik Command # # 1) Find KlustaKwik executable # mclust_path = Path('C:/Users/reichler/src/MClustPipeline/MClust/KlustaKwik') # pf_system = platform.system() # logger.debug(f'Platform: {pf_system}') # if pf_system == 'Linux': # kk_executable = mclust_path / cfg['KLUSTAKWIK_PATH_LINUX'] # elif pf_system == 'Windows': # kk_executable = mclust_path / cfg['KLUSTAKWIK_PATH_WINDOWS'] # else: # raise NotImplemented(f'No KlustaKwik executable defined for platform {pf_system}') # logger.debug(kk_executable) kk_executable = cli_args.KK # 2) Find target file stem working_dir = Path(cli_args.target).resolve() if working_dir.is_file() and working_dir.exists(): tetrode_files = [working_dir.name] working_dir = working_dir.parent logger.debug(f'Using single file mode with {str(tetrode_files[0])}') else: tetrode_files = sorted( [tf.name for tf in working_dir.glob(cfg['TARGET_FILE_GLOB'])]) logger.debug(f'Working dir: {working_dir}') # No parallel/serial execution supported right now if len(tetrode_files) > 1: raise NotImplemented( 'Currently only one target file per call supported!') logger.debug(f'Target found: {tetrode_files}') tetrode_file_stem = str(tetrode_files[0]).split(".")[0] tetrode_file_elecno = tetrode_files[0].split(".")[-1] # 3) Check if output file already exists clu_file = (working_dir / tetrode_file_stem).with_suffix(f'.clu.{tetrode_file_elecno}') if clu_file.exists() and not (cli_args.force or cli_args.skip): raise FileExistsError( 'Clu file already exists. Use --force to overwrite.') # 4) combine executable and arguments kk_cmd = f'{kk_executable} {tetrode_file_stem} -ElecNo {tetrode_file_elecno}' kk_cmd_list = kk_cmd.split(' ') logger.debug(f'KK COMMAND: {kk_cmd}') logger.debug(f'KK COMMAND LIST: {kk_cmd_list}') # Call KlustaKwik and gather output # TODO: Use communicate to interact with KK, i.e. write to log and monitor progress # see https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging logger.info('Starting KlustaKwik process') if cfg['PRINT_KK_OUTPUT']: stdout = subprocess.STDOUT else: stdout = subprocess.PIPE # EXECUTE KLUSTAKWIK if not clu_file.exists() or cli_args.force: kk_call = subprocess.run(kk_cmd_list, stderr=subprocess.PIPE, stdout=stdout) kk_error = kk_call.returncode logger.debug('Writing KlustaKwik log file') with open(clu_file.with_suffix('.log'), 'w') as log_file: log_file.write(kk_call.stderr.decode('ascii')) # Check call return code and output if kk_error: logging.error(f'KlustaKwik error code: {kk_error}') exit(kk_error) else: logging.debug(f'KlustaKwik successful: {kk_error}') # Load clu file logger.debug(f'Loading {clu_file}') clu_df = pd.read_csv(clu_file, dtype='category', names=['cluster_id'], skiprows=1) cluster_labels = clu_df['cluster_id'].cat.categories num_clusters = len(cluster_labels) logger.info(f'{len(clu_df)} spikes in {num_clusters} clusters') # Find all feature .fd files feature_files = list(working_dir.glob(tetrode_file_stem + '_*.fd')) ff_sizes = [ff.stat().st_mtime for ff in feature_files] feature_files = [f for t, f in sorted(zip(ff_sizes, feature_files))] if not len(feature_files): raise FileNotFoundError(f'No Feature Files found in {working_dir}') # TODO: Stupid, the feature names are in the .fd file already feature_names = [ str(ff.name).split(tetrode_file_stem + '_')[1].split('.')[0] for ff in feature_files ] logger.info(f'Loading features: {feature_names}') color_keys = cfg['CLUSTER_COLORS'] with open(clu_file.with_suffix('.html'), 'w') as crf: crf.write('<head></head><body><h1>{}</h1>'.format(clu_file.name)) for fd_file, fet_name in zip(feature_files, feature_names): crf.write('<h3>Feature: {}</h3>\n'.format(fet_name)) logger.info(f'Generating images for feature {fet_name}') if not fd_file.exists(): continue logger.debug(f'Loading {fd_file}') mat_fet = h5s.loadmat(str(fd_file), appendmat=False) fd_df = pd.DataFrame(mat_fet['FeatureData']) fd_df.rename(columns={c: str(c) for c in fd_df.columns}, inplace=True) if not len(clu_df) == len(fd_df): raise ValueError( f'Number of cluster labels ({num_clusters}) does not match number of spikes' f'in {fd_file} ({len(fd_df)})') fd_df['clu_id'] = clu_df.cluster_id.astype('category') logger.debug( f'Feature {fet_name} loaded with {len(fd_df)} spikes, {fd_df.shape[1] - 1} dimensions ' ) images = [] titles = [] for cc in combinations(map(str, range(len(fd_df.columns) - 1)), r=2): fet_title = f'{fet_name}:{cc[1]} vs {fet_name}:{cc[0]}' x_range = (np.percentile(fd_df[cc[0]], 0.01), np.percentile(fd_df[cc[0]], 99.9)) y_range = (np.percentile(fd_df[cc[1]], 0.01), np.percentile(fd_df[cc[1]], 99.9)) logger.debug( f'shading {len(fd_df)} points in {fd_df.shape[1] - 1} dimensions' ) canvas = ds.Canvas(plot_width=300, plot_height=300, x_range=x_range, y_range=y_range) agg = canvas.points(fd_df, x=cc[0], y=cc[1], agg=ds.count_cat('clu_id')) with np.errstate(invalid='ignore'): img = ds_tf.shade(agg, how='log', color_key=color_keys) img = img if cli_args.no_spread else ds_tf.spread(img, px=1) images.append(img) titles.append(fet_title) logger.debug(f'Creating plot for {fet_name}') fet_fig = ds_plot_features(images, how='log', fet_titles=titles) crf.write(fig2html(fet_fig) + '</br>\n') plt.close(fet_fig)
def nodesplot(nodes, name=None, canvas=None, cat=None): canvas = ds.Canvas(**cvsopts) if canvas is None else canvas aggregator=None if cat is None else ds.count_cat(cat) agg=canvas.points(nodes,'x','y',aggregator) return tf.spread(tf.shade(agg, cmap=["#FF3333"]), px=3, name=name)
layout = go.Layout( xaxis = dict(title = 'Year Built'), yaxis = dict(title = 'Average Number Of FLoors') ) fig = go.Figure(data = [trace], layout = layout) py.iplot(fig, filename = 'AvgNumOfFloor') assess =ny[['AssessTot', 'AssessLand','lon','lat']].copy() assess['AssBuilding'] = assess['AssessTot'].sub(assess['AssessLand'], axis=0) # Subtract to get Building Value labels = ['A', 'B', 'C'] assess['AssLandBins'] = pd.qcut(assess.AssessLand, 3, labels=labels) assess['AssBuldBins'] = pd.qcut(assess.AssBuilding, 3, labels=labels) assess['Comb'] = assess.AssLandBins.astype(str) + '_' + assess.AssBuldBins.astype(str) assess.Comb = assess.Comb.astype('category') colors = {'A_A': '#e8a8e8', 'A_B': '#a4acac', 'A_C': '#c85a5a', 'B_A': '#b0a5df', 'B_B': '#ad9ea5', 'B_C': '#c85356', 'C_A': '#64ccbe', 'C_B': '#a27f8c', 'C_C': '#c74249'} NewYorkCity = (( -74.29, -73.69), (40.49, 40.92)) cvs = ds.Canvas(700, 700, * NewYorkCity) agg = cvs.points(assess, 'lon', 'lat', ds.count_cat('Comb')) view = tf.shade(agg, color_key = colors) export(tf.spread(view, px=1), 'Cloropleth')
def run_kk(params, run_kk=True): cfg,maxc, target_path = params tt_fname = target_path.name tetrode_file_stem = tt_fname.split(".")[0] tetrode_file_elecno = tt_fname.split(".")[-1] working_dir = target_path.parent logging.debug(f'Tetrode name: {tt_fname}, stem: {tetrode_file_stem}, ElecNo: {tetrode_file_elecno}') clu_file = working_dir / (tetrode_file_stem + f'.clu.{tetrode_file_elecno}') if clu_file.exists() and cfg['skip']: logging.error(f'Clu file {clu_file} exists. Skipping.') run_kk = False # Read in feature validity validity_path = target_path.with_suffix('.validity') if not validity_path.exists(): logger.warning('No explicit feature validity given, falling back to default = all used.') with open(validity_path) as vfp: validity_string = vfp.readline() logger.debug(f'Channel validity: {validity_string}') # Combine executable and arguments kk_executable = cfg["kk_executable"] kk_cmd = f'{kk_executable} {tetrode_file_stem} -ElecNo {tetrode_file_elecno} -UseFeatures {validity_string} -MaxPossibleClusters {maxc}' if cfg['KKv3']: kk_cmd += ' -UseDistributional 0' kk_cmd_list = kk_cmd.split(' ') logger.debug(f'KK COMMAND: {kk_cmd}') logger.debug(f'KK COMMAND LIST: {kk_cmd_list}') # Call KlustaKwik and gather output # TODO: Use communicate to interact with KK, i.e. write to log and monitor progress # see https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging logger.info('Starting KlustaKwik process') if cfg['PRINT_KK_OUTPUT']: stdout = None else: stdout = subprocess.PIPE if run_kk: kk_call = subprocess.run(kk_cmd_list, stderr=subprocess.STDOUT, stdout=stdout) kk_error = kk_call.returncode logger.debug('Writing KlustaKwik log file') logger.debug('Clu File: ' + str(clu_file)) if kk_call.stdout is not None: with open(clu_file.with_suffix('.log'), 'w') as log_file: log_file.write(kk_call.stdout.decode('ascii')) else: logging.warning('Missing stdout, not writing log file!') # Check call return code and output if kk_error: logging.error(f'KlustaKwik error code: {kk_error}') exit(kk_error) else: logging.debug(f'KlustaKwik successful: {kk_error}') # Load clu file logger.debug(f'Loading {clu_file}') clu_df = pd.read_csv(clu_file, dtype='category', names=['cluster_id'], skiprows=1) cluster_labels = clu_df['cluster_id'].cat.categories num_clusters = len(cluster_labels) logger.info(f'{len(clu_df)} spikes in {num_clusters} clusters') # Find all feature .fd files feature_files = list(working_dir.glob(tetrode_file_stem + '_*.fd')) ff_sizes = [ff.stat().st_mtime for ff in feature_files] feature_files = [f for t, f in sorted(zip(ff_sizes, feature_files))] if not len(feature_files): raise FileNotFoundError(f'No Feature Files found in {working_dir}') # TODO: Stupid, the feature names are in the .fd file already feature_names = [str(ff.name).split(tetrode_file_stem + '_')[1].split('.')[0] for ff in feature_files] logger.info(f'Loading features: {feature_names}') color_keys = cfg['CLUSTER_COLORS'] with open(clu_file.with_suffix('.html'), 'w') as crf: crf.write('<head></head><body><h1>{}</h1>'.format(clu_file.name)) for fd_file, fet_name in zip(feature_files, feature_names): crf.write('<h3>Feature: {}</h3>\n'.format(fet_name)) logger.info(f'Generating images for feature {fet_name}') if not fd_file.exists(): continue logger.debug(f'Loading {fd_file}') mat_fet = h5s.loadmat(str(fd_file), appendmat=False) fd_df = pd.DataFrame(mat_fet['FeatureData']) fd_df.rename(columns={c: str(c) for c in fd_df.columns}, inplace=True) if not len(clu_df) == len(fd_df): raise ValueError(f'Number of cluster labels ({num_clusters}) does not match number of spikes' f'in {fd_file} ({len(fd_df)})') fd_df['clu_id'] = clu_df.cluster_id.astype('category') logger.debug(f'Feature {fet_name} loaded with {len(fd_df)} spikes, {fd_df.shape[1] - 1} dimensions ') images = [] titles = [] for cc in combinations(map(str, range(len(fd_df.columns) - 1)), r=2): fet_title = f'x: {fet_name}:{cc[0]} vs y: {fet_name}:{cc[1]}' x_range = (np.percentile(fd_df[cc[0]], 0.01), np.percentile(fd_df[cc[0]], 99.9)) y_range = (np.percentile(fd_df[cc[1]], 0.01), np.percentile(fd_df[cc[1]], 99.9)) logger.debug(f'shading {len(fd_df)} points in {fd_df.shape[1] - 1} dimensions') canvas = ds.Canvas(plot_width=300, plot_height=300, x_range=x_range, y_range=y_range) try: agg = canvas.points(fd_df, x=cc[0], y=cc[1], agg=ds.count_cat('clu_id')) with np.errstate(invalid='ignore'): img = ds_tf.shade(agg, how='log', color_key=color_keys) img = img if cfg['no_spread'] else ds_tf.spread(img, px=1) except ZeroDivisionError: img = None images.append(img) titles.append(fet_title) logger.debug(f'Creating plot for {fet_name}') fet_fig = ds_plot_features(images, how='log', fet_titles=titles) crf.write(fig2html(fet_fig) + '</br>\n') plt.close(fet_fig)
def test_float32_spread(): data = np.array( [[1, 1, np.nan, np.nan, np.nan], [1, 2, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, 3, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan]], dtype='float32') coords = [np.arange(5), np.arange(5)] arr = xr.DataArray(data, coords=coords, dims=dims) s = tf.spread(arr) o = np.array([[5, 5, 3, np.nan, np.nan], [5, 5, 3, np.nan, np.nan], [3, 3, 5, 3, 3], [np.nan, np.nan, 3, 3, 3], [np.nan, np.nan, 3, 3, 3]]) np.testing.assert_equal(s.data, o) assert (s.x_axis == arr.x_axis).all() assert (s.y_axis == arr.y_axis).all() assert s.dims == arr.dims s = tf.spread(arr, px=2) o = np.array([[5, 5, 5, 3, np.nan], [5, 5, 8, 6, 3], [5, 8, 7, 5, 3], [3, 6, 5, 3, 3], [np.nan, 3, 3, 3, 3]]) np.testing.assert_equal(s.data, o) s = tf.spread(arr, shape='square') o = np.array([[5, 5, 3, np.nan, np.nan], [5, 5, 3, np.nan, np.nan], [3, 3, 5, 3, 3], [np.nan, np.nan, 3, 3, 3], [np.nan, np.nan, 3, 3, 3]]) np.testing.assert_equal(s.data, o) s = tf.spread(arr, how='min') o = np.array([[1, 1, 1, np.nan, np.nan], [1, 1, 1, np.nan, np.nan], [1, 1, 2, 3, 3], [np.nan, np.nan, 3, 3, 3], [np.nan, np.nan, 3, 3, 3]]) np.testing.assert_equal(s.data, o) s = tf.spread(arr, how='max') o = np.array([[2, 2, 2, np.nan, np.nan], [2, 2, 2, np.nan, np.nan], [2, 2, 3, 3, 3], [np.nan, np.nan, 3, 3, 3], [np.nan, np.nan, 3, 3, 3]]) np.testing.assert_equal(s.data, o) mask = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]]) data = np.array([[np.nan, np.nan, np.nan, 1, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, 1, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan]], dtype='float32') arr = xr.DataArray(data, coords=coords, dims=dims) s = tf.spread(arr, mask=mask) o = np.array([[0, 0, 0, 1, 0], [1, 0, 2, 0, 1], [0, 1, 0, 0, 0], [1, 0, 1, 0, 0], [0, 0, 0, 0, 0]]) o = np.array([[np.nan, np.nan, np.nan, 1, np.nan], [1, np.nan, 2, np.nan, 1], [np.nan, 1, np.nan, np.nan, np.nan], [1, np.nan, 1, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan]]) np.testing.assert_equal(s.data, o) s = tf.spread(arr, px=0) np.testing.assert_equal(s.data, arr.data) pytest.raises(ValueError, lambda: tf.spread(arr, px=-1)) pytest.raises(ValueError, lambda: tf.spread(arr, mask=np.ones(2))) pytest.raises(ValueError, lambda: tf.spread(arr, mask=np.ones((2, 2))))
trees['y_sp'].values) trees = trees[(trees['lon'] < -60) & (trees['lon'] > -100) & (trees['lat'] < 60) & (trees['lat'] > 20)] #Also using project 2 code to get a map locating the trees. #Defining some helper functions for DataShader background = "black" export = partial(export_image, background=background, export_path="export") cm = partial(colormap_select, reverse=(background != "black")) NewYorkCity = ((-74.29, -73.69), (40.49, 40.92)) cvs = ds.Canvas(700, 700, *NewYorkCity) agg = cvs.points(trees, 'lon', 'lat') view = tf.shade(agg, cmap=cm(viridis), how='log') export(tf.spread(view, px=2), 'trees') agg = cvs.points(trees, 'lon', 'lat', ds.mean('health')) view = tf.shade(agg, cmap=cm(viridis), how='eq_hist') export(tf.spread(view, px=2), 'trees_health') #%% """ The block below is the actual Dash App. It simply displays the images with the user inputs defined by Radio bottons. It then subsets the dataframe and displays the appropreiate histogram. """ #%% # https://www.youtube.com/watch?v=wv2MXJIdKRY #https://dash.plot.ly/getting-started-part-2 app = dash.Dash()
def nodesplot(nodes, name=None, canvas=None, cat=None): canvas = ds.Canvas(**cvsopts) if canvas is None else canvas aggregator = None if cat is None else ds.count_cat(cat) agg = canvas.points(nodes, 'x', 'y', aggregator) return tf.spread(tf.shade(agg, cmap=["#FF3333"]), px=3, name=name)
# This shows us the distribution, but it's subject to some biases discussed in the Anaconda notebook [Plotting Perils](https://anaconda.org/jbednar/plotting_pitfalls/notebook). # # Here is what the same plot would look like in datashader: # # # In[18]: cvs = ds.Canvas(800, 500, x_range=(ny['yearbuilt'].min(), ny['yearbuilt'].max()), y_range=(ny['numfloors'].min(), ny['numfloors'].max())) agg = cvs.points(ny, 'yearbuilt', 'numfloors') view = tf.shade(agg, cmap=cm(Greys9), how='log') export(tf.spread(view, px=2), 'yearvsnumfloors') # That's technically just a scatterplot, but the points are smartly placed and colored to mimic what one gets in a heatmap. Based on the pixel size, it will either display individual points, or will color the points of denser regions. # # Datashader really shines when looking at geographic information. Here are the latitudes and longitudes of our dataset plotted out, giving us a map of the city colored by density of structures: # In[10]: NewYorkCity = ((-74.29, -73.69), (40.49, 40.92)) cvs = ds.Canvas(700, 700, *NewYorkCity) agg = cvs.points(ny, 'lon', 'lat') view = tf.shade(agg, cmap=cm(inferno), how='log') export(tf.spread(view, px=2), 'firery') # Interestingly, since we're looking at structures, the large buildings of Manhattan show up as less dense on the map. The densest areas measured by number of lots would be single or multi family townhomes. #
def show_velphase(ds, ray_df, ray_start, ray_end, triray, filename): # take in the yt dataset (ds) and a ray as a dataframe # preliminaries rs = ray_start.ndarray_view() re = ray_end.ndarray_view() imsize = 500 core_width = 10. proper_box_size = ds.get_parameter( 'CosmologyComovingBoxSize') / ds.get_parameter( 'CosmologyHubbleConstantNow') * 1000. # in kpc redshift = ds.get_parameter('CosmologyCurrentRedshift') # take out a "core sample" that extends along the ray with a width given by core_width ad = ds.r[rs[0]:re[0], rs[1] - 0.5 * core_width / proper_box_size:rs[1] + 0.5 * core_width / proper_box_size, rs[2] - 0.5 * core_width / proper_box_size:rs[2] + 0.5 * core_width / proper_box_size] cell_vol = ad["cell_volume"] cell_mass = ad["cell_mass"] cell_size = np.array(cell_vol)**(1. / 3.) * proper_box_size x_cells = ad['x'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) y_cells = ad['y'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) z_cells = ad['z'].ndarray_view( ) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. ) dens = np.log10(ad['density'].ndarray_view()) temp = np.log10(ad['temperature'].ndarray_view()) phase = np.chararray(np.size(temp), 4) phase[temp < 19.] = 'hot' phase[temp < 6.] = 'warm' phase[temp < 5.] = 'cool' phase[temp < 4.] = 'cold' df = pd.DataFrame({ 'x': x_cells, 'y': y_cells, 'z': z_cells, 'vx': ad["x-velocity"], 'vy': ad["y-velocity"], 'vz': ad["z-velocity"], 'temp': temp, 'dens': dens, 'phase': phase }) df.phase = df.phase.astype('category') cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(np.min(df['x']), np.max(df['x'])), y_range=(np.mean(df['y']) - 100. / 0.695, np.mean(df['y']) + 100. / 0.695)) agg = cvs.points(df, 'x', 'y', dshader.count_cat('phase')) img = tf.shade(agg, color_key=phase_color_key) x_y = tf.spread(img, px=2, shape='square') x_y.to_pil().save(filename + '_x_vs_y.png') cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(np.min(df['x']), np.max(df['x'])), y_range=(-0.008, 0.008)) agg = cvs.points(df, 'x', 'vx', dshader.count_cat('phase')) img = tf.shade(agg, color_key=phase_color_key) x_vx = tf.spread(img, px=1, shape='square') x_vx.to_pil().save(filename + '_x_vs_vx.png') for species in species_dict.keys(): cvs = dshader.Canvas(plot_width=imsize, plot_height=imsize, x_range=(rs[0], re[0]), y_range=(-0.008, 0.008)) vx = tf.shade(cvs.points(ray_df, 'x', 'x-velocity', agg=reductions.mean(species_dict[species])), how='eq_hist') tf.set_background(vx, "white") ray_vx = tf.spread(vx, px=4, shape='square') pil = ray_vx.to_pil() pil.save(filename + '_' + species + '_ray_vx.png', format='png')