def test_dynspread():
b = 0xffff0000
data = np.array([[b, b, 0, 0, 0],
[b, b, 0, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, b, 0],
[0, 0, 0, 0, 0]], dtype='uint32')
coords = [np.arange(5), np.arange(5)]
img = tf.Image(data, coords=coords, dims=dims)
assert tf.dynspread(img).equals(tf.spread(img, 1))
assert tf.dynspread(img, threshold=0.9).equals(tf.spread(img, 2))
assert tf.dynspread(img, threshold=0).equals(img)
assert tf.dynspread(img, max_px=0).equals(img)
pytest.raises(ValueError, lambda: tf.dynspread(img, threshold=1.1))
pytest.raises(ValueError, lambda: tf.dynspread(img, max_px=-1))
def render_image(self):
pix = tf.shade(self.agg, cmap=self.color_ramp, color_key=self.colormap, how=self.transfer_function)
if self.spread_size > 0:
pix = tf.spread(pix, px=self.spread_size)
return pix
def test_categorical_dynspread():
a_data = np.array([[1, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0],
[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]],
dtype='int32')
b_data = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [1, 0, 0, 0, 0],
[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]],
dtype='int32')
c_data = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1],
[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]],
dtype='int32')
data = np.dstack([a_data, b_data, c_data])
coords = [np.arange(5), np.arange(5)]
arr = xr.DataArray(data,
coords=coords + [['a', 'b', 'c']],
dims=dims + ['cat'])
assert tf.dynspread(arr, threshold=0.4).equals(tf.spread(arr, 0))
assert tf.dynspread(arr, threshold=0.7).equals(tf.spread(arr, 1))
assert tf.dynspread(arr, threshold=1.0).equals(tf.spread(arr, 3))
assert tf.dynspread(arr, max_px=0).equals(arr)
def viewInteractiveImage(x_range, y_range, w, h, data_source,
**kwargs):
dd = data_source[[self.x, self.y, self.aggregate_col]]
dd[self.x] = self._to_xaxis_type(dd[self.x])
dd[self.y] = self._to_yaxis_type(dd[self.y])
x_range = self._to_xaxis_type(x_range)
y_range = self._to_yaxis_type(y_range)
cvs = ds.Canvas(plot_width=w,
plot_height=h,
x_range=x_range,
y_range=y_range)
aggregator, cmap = _compute_datashader_assets(
dd,
self.x,
self.aggregate_col,
self.aggregate_fn,
self.color_palette,
)
agg = cvs.points(
dd,
self.x,
self.y,
aggregator,
)
if self.constant_limit is None or self.aggregate_fn == "count":
self.constant_limit = [
float(cp.nanmin(agg.data)),
float(cp.nanmax(agg.data)),
]
self.render_legend()
span = {"span": self.constant_limit}
if self.pixel_shade_type == "eq_hist":
span = {}
img = tf.shade(agg, how=self.pixel_shade_type, **cmap, **span)
if self.pixel_spread == "dynspread":
return tf.dynspread(
img,
threshold=self.pixel_density,
max_px=self.point_size,
shape=self.point_shape,
)
else:
return tf.spread(img,
px=self.point_size,
shape=self.point_shape)
def render_image(self):
if self.colormaps:
colormap = self.colormaps.get(self.field_title, None)
if colormap:
self.colormap = colormap
self.colornames = self.color_name_maps[self.field_title]
pix = tf.shade(self.agg, cmap=self.color_ramp, color_key=self.colormap, how=self.transfer_function)
if self.spread_size > 0:
pix = tf.spread(pix, px=self.spread_size)
return pix
def plot_values(data_frame):
canvas = ds.Canvas(plot_width=1000,
plot_height=1000,
x_range=(0, 1),
y_range=(0, 1))
agg = canvas.points(data_frame, 'B2', 'B5')
image = tf.shade(agg,
cmap=['lightblue', 'darkblue'],
how='linear',
alpha=150)
image = tf.spread(image, px=1)
image = tf.set_background(image, color='#222222')
return image
def render_image(self):
# handle categorical field
if self.field in self.categorical_fields:
pix = tf.colorize(self.agg, self.colormap, how=self.transfer_function)
# handle ordinal field
elif self.field in self.ordinal_fields:
pix = tf.interpolate(self.agg, cmap=self.color_ramp, how=self.transfer_function)
# handle no field
else:
pix = tf.interpolate(self.agg, cmap=self.color_ramp, how=self.transfer_function)
if self.spread_size > 0:
pix = tf.spread(pix, px=self.spread_size)
return pix
def create_datashader_tile(gdf, x, y, zoom):
tile = mercantile.Tile(x, y, zoom)
bounds = mercantile.bounds(tile)
geom = shapely.geometry.box(*bounds)
mask = gdf.within(geom)
xy_bounds = mercantile.xy_bounds(tile)
cvs = ds.Canvas(plot_width=256,
plot_height=256,
x_range=(xy_bounds[0], xy_bounds[2]),
y_range=(xy_bounds[1], xy_bounds[3]))
agg = cvs.points(gdf[mask], 'x', 'y', agg=ds.count())
img = tf.shade(agg, cmap=['blue', 'darkblue', 'black'], how='eq_hist')
img = tf.spread(img, name="spread 1px")
return img.to_pil()
def master_tile(data, img, opts, z):
"""
makes the map at zoom level = z to be broken down into tiles
:param zoom_level:
:return:
"""
dim = map_size(z)
proj_data = proj_factor(z, img) * data
proj_data.y = dim - proj_data.y
scene = ds.Canvas(x_range=[0, dim], y_range=[0, dim], plot_width=dim, plot_height=dim)
aggregation = scene.points(proj_data, 'x', 'y')
alpha = opts['dot_alpha'](z)
image = tf.shade(aggregation, cmap=opts['dot_color'], alpha=alpha)
if opts['dot_size']:
image = tf.spread(image, px=opts['dot_size'], shape='circle', name="spread square")
export_image(image, 'master_tile', background=None)
return image
def serve_image(): #dataset
# parse params
print('-------------- server receives request-----------')
width = int(request.args.get('width'))
height = int(request.args.get('height'))
spread = int(request.args.get('spread'))
xmin = float(request.args.get('xmin'))
xmax = float(request.args.get('xmax'))
ymax = float(request.args.get('ymax'))
ymin = float(request.args.get('ymin'))
colorw = str(request.args.get('colorw'))
colorb = str(request.args.get('colorb'))
colora = str(request.args.get('colora'))
colorh = str(request.args.get('colorh'))
coloro = str(request.args.get('coloro'))
x_range = webm(latitude=ymin, longitude=xmin)
y_range = webm(latitude=ymax, longitude=xmax)
cvs = ds.Canvas(plot_width=width,
plot_height=height,
x_range=(x_range[0], y_range[0]),
y_range=(x_range[1], y_range[1]))
agg = cvs.points(df, 'easting', 'northing', ds.count_cat('race'))
color_key = {
'w': colorw,
'b': colorb,
'a': colora,
'h': colorh,
'o': coloro
}
img = tf.shade(agg, color_key=color_key, how='eq_hist')
img = tf.spread(img, px=spread)
img_io = img.to_bytesio()
return send_file(img_io, mimetype='image/png')
def render_image(self):
# handle categorical field
if self.field in self.categorical_fields:
pix = tf.colorize(self.agg,
self.colormap,
how=self.transfer_function)
# handle ordinal field
elif self.field in self.ordinal_fields:
pix = tf.interpolate(self.agg,
cmap=self.color_ramp,
how=self.transfer_function)
# handle no field
else:
pix = tf.interpolate(self.agg,
cmap=self.color_ramp,
how=self.transfer_function)
if self.spread_size > 0:
pix = tf.spread(pix, px=self.spread_size)
return pix
def viewInteractiveImage(x_range, y_range, w, h, data_source):
cvs = cds.Canvas(
plot_width=w, plot_height=h, x_range=x_range, y_range=y_range
)
agg = cvs.points(
data_source,
self.x,
self.y,
getattr(cds, self.aggregate_fn)(self.aggregate_col),
)
img = tf.shade(
agg, cmap=self.color_palette, how=self.pixel_shade_type
)
if self.pixel_spread == "dynspread":
return tf.dynspread(
img,
threshold=self.pixel_density,
max_px=self.point_size,
shape=self.point_shape,
)
else:
return tf.spread(
img, px=self.point_size, shape=self.point_shape
)
def test_spread():
p = 0x7d00007d
g = 0x7d00FF00
b = 0x7dFF0000
data = np.array([[p, p, 0, 0, 0],
[p, g, 0, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, b, 0],
[0, 0, 0, 0, 0]], dtype='uint32')
coords = [np.arange(5), np.arange(5)]
img = tf.Image(data, coords=coords, dims=dims)
s = tf.spread(img)
o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]])
np.testing.assert_equal(s.data, o)
assert (s.x_axis == img.x_axis).all()
assert (s.y_axis == img.y_axis).all()
assert s.dims == img.dims
s = tf.spread(img, px=2)
o = np.array([[0xed00863b, 0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000],
[0xed00863b, 0xed00863b, 0xf581411c, 0xdc904812, 0x7dff0000],
[0xed00863b, 0xf581411c, 0xed864419, 0xbca85600, 0x7dff0000],
[0xbc00a82a, 0xdc904812, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000, 0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, shape='square')
o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, how='add')
o = np.array([[0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000],
[0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000],
[0xfa007f3e, 0xfa007f3e, 0xfa7f7f00, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000]])
np.testing.assert_equal(s.data, o)
mask = np.array([[1, 0, 1],
[0, 1, 0],
[1, 0, 1]])
s = tf.spread(img, mask=mask)
o = np.array([[0xbc00a82a, 0xbc00007d, 0x7d00ff00, 0x00000000, 0x00000000],
[0xbc00007d, 0xbc00a82a, 0x7d00007d, 0x00000000, 0x00000000],
[0x7d00ff00, 0x7d00007d, 0xbca85600, 0x00000000, 0x7dff0000],
[0x00000000, 0x00000000, 0x00000000, 0x7dff0000, 0x00000000],
[0x00000000, 0x00000000, 0x7dff0000, 0x00000000, 0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, px=0)
np.testing.assert_equal(s.data, img.data)
pytest.raises(ValueError, lambda: tf.spread(img, px=-1))
pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones(2)))
pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones((2, 2))))
def my_nodesplot(nodes, name=None, canvas=None, cat=None):
canvas = ds.Canvas(**cvsopts) if canvas is None else canvas
aggregator=None if cat is None else ds.count_cat(cat)
agg=canvas.points(nodes,'x','y',aggregator)
return tf.spread(tf.shade(agg, cmap=["#333333"], color_key=colors, min_alpha=255), px=3, name=name)
def _apply_spreading(self, array):
img = tf.Image(array)
return tf.spread(img, px=self.p.px,
how=self.p.how, shape=self.p.shape).data
def main(args):
parser = argparse.ArgumentParser('Clustering with KlustaKwik')
parser.add_argument(
'target',
help=
'Target path, either path containing tetrode files, or single tetrodeXX.fet.0'
)
parser.add_argument('--KK', help='Path to KlustaKwik executable')
parser.add_argument('--features',
nargs='*',
help='list of features to use for clustering')
parser.add_argument('--config', help='Path to configuration file')
parser.add_argument('--cluster', action='store_true', help='Directly run ')
parser.add_argument('--force',
help='Overwrite existing files.',
action='store_true')
parser.add_argument('--skip',
help='Skip if clu file exists already',
action='store_true')
parser.add_argument('--no_spread',
help='Shade report plots without static spread',
action='store_true')
cli_args = parser.parse_args(args)
# Load default configuration yaml file
default_cfg_path = Path(
pkg_resources.resource_filename(
__name__, '../resources/cluster_defaults.yml')).resolve()
if not default_cfg_path.exists():
logging.error('Could not find default config file.')
raise FileNotFoundError
logger.debug('Loading default configuration')
cfg = load_yaml(default_cfg_path)
# Load local config file if it exists
local_cfg_path = Path(
pkg_resources.resource_filename(
__name__, '../resources/cluster_defaults_local.yml')).resolve()
if local_cfg_path.exists():
logger.debug('Loading and updating with local configuration')
local_cfg = load_yaml(local_cfg_path)
cfg.update(local_cfg)
# Load custom config path
custom_cfg_path = Path(
cli_args.config).resolve() if cli_args.config else None
if custom_cfg_path:
if custom_cfg_path.exists():
logger.debug('Loading and updating with custom configuration')
cfg.update(load_yaml(custom_cfg_path))
else:
raise FileNotFoundError(
f"Could not load configuration file {custom_cfg_path}")
# Load parameters from command line
logger.debug('Parsing and updating configuration with CLI arguments')
cfg.update(vars(cli_args))
logger.debug(cfg)
# try to find Klustakwik executable if necessary...
if cli_args.KK is None:
cli_args.KK = shutil.which('KlustaKwik') or shutil.which(
'klustakwik') or shutil.which('Klustakwik')
if cli_args.KK is None and cli_args.cluster:
raise FileNotFoundError(
'Could not find the KlustaKwik executable on the path, and none given.'
)
# # Building KlustaKwik Command
# # 1) Find KlustaKwik executable
# mclust_path = Path('C:/Users/reichler/src/MClustPipeline/MClust/KlustaKwik')
# pf_system = platform.system()
# logger.debug(f'Platform: {pf_system}')
# if pf_system == 'Linux':
# kk_executable = mclust_path / cfg['KLUSTAKWIK_PATH_LINUX']
# elif pf_system == 'Windows':
# kk_executable = mclust_path / cfg['KLUSTAKWIK_PATH_WINDOWS']
# else:
# raise NotImplemented(f'No KlustaKwik executable defined for platform {pf_system}')
# logger.debug(kk_executable)
kk_executable = cli_args.KK
# 2) Find target file stem
working_dir = Path(cli_args.target).resolve()
if working_dir.is_file() and working_dir.exists():
tetrode_files = [working_dir.name]
working_dir = working_dir.parent
logger.debug(f'Using single file mode with {str(tetrode_files[0])}')
else:
tetrode_files = sorted(
[tf.name for tf in working_dir.glob(cfg['TARGET_FILE_GLOB'])])
logger.debug(f'Working dir: {working_dir}')
# No parallel/serial execution supported right now
if len(tetrode_files) > 1:
raise NotImplemented(
'Currently only one target file per call supported!')
logger.debug(f'Target found: {tetrode_files}')
tetrode_file_stem = str(tetrode_files[0]).split(".")[0]
tetrode_file_elecno = tetrode_files[0].split(".")[-1]
# 3) Check if output file already exists
clu_file = (working_dir /
tetrode_file_stem).with_suffix(f'.clu.{tetrode_file_elecno}')
if clu_file.exists() and not (cli_args.force or cli_args.skip):
raise FileExistsError(
'Clu file already exists. Use --force to overwrite.')
# 4) combine executable and arguments
kk_cmd = f'{kk_executable} {tetrode_file_stem} -ElecNo {tetrode_file_elecno}'
kk_cmd_list = kk_cmd.split(' ')
logger.debug(f'KK COMMAND: {kk_cmd}')
logger.debug(f'KK COMMAND LIST: {kk_cmd_list}')
# Call KlustaKwik and gather output
# TODO: Use communicate to interact with KK, i.e. write to log and monitor progress
# see https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
logger.info('Starting KlustaKwik process')
if cfg['PRINT_KK_OUTPUT']:
stdout = subprocess.STDOUT
else:
stdout = subprocess.PIPE
# EXECUTE KLUSTAKWIK
if not clu_file.exists() or cli_args.force:
kk_call = subprocess.run(kk_cmd_list,
stderr=subprocess.PIPE,
stdout=stdout)
kk_error = kk_call.returncode
logger.debug('Writing KlustaKwik log file')
with open(clu_file.with_suffix('.log'), 'w') as log_file:
log_file.write(kk_call.stderr.decode('ascii'))
# Check call return code and output
if kk_error:
logging.error(f'KlustaKwik error code: {kk_error}')
exit(kk_error)
else:
logging.debug(f'KlustaKwik successful: {kk_error}')
# Load clu file
logger.debug(f'Loading {clu_file}')
clu_df = pd.read_csv(clu_file,
dtype='category',
names=['cluster_id'],
skiprows=1)
cluster_labels = clu_df['cluster_id'].cat.categories
num_clusters = len(cluster_labels)
logger.info(f'{len(clu_df)} spikes in {num_clusters} clusters')
# Find all feature .fd files
feature_files = list(working_dir.glob(tetrode_file_stem + '_*.fd'))
ff_sizes = [ff.stat().st_mtime for ff in feature_files]
feature_files = [f for t, f in sorted(zip(ff_sizes, feature_files))]
if not len(feature_files):
raise FileNotFoundError(f'No Feature Files found in {working_dir}')
# TODO: Stupid, the feature names are in the .fd file already
feature_names = [
str(ff.name).split(tetrode_file_stem + '_')[1].split('.')[0]
for ff in feature_files
]
logger.info(f'Loading features: {feature_names}')
color_keys = cfg['CLUSTER_COLORS']
with open(clu_file.with_suffix('.html'), 'w') as crf:
crf.write('<head></head><body><h1>{}</h1>'.format(clu_file.name))
for fd_file, fet_name in zip(feature_files, feature_names):
crf.write('<h3>Feature: {}</h3>\n'.format(fet_name))
logger.info(f'Generating images for feature {fet_name}')
if not fd_file.exists():
continue
logger.debug(f'Loading {fd_file}')
mat_fet = h5s.loadmat(str(fd_file), appendmat=False)
fd_df = pd.DataFrame(mat_fet['FeatureData'])
fd_df.rename(columns={c: str(c)
for c in fd_df.columns},
inplace=True)
if not len(clu_df) == len(fd_df):
raise ValueError(
f'Number of cluster labels ({num_clusters}) does not match number of spikes'
f'in {fd_file} ({len(fd_df)})')
fd_df['clu_id'] = clu_df.cluster_id.astype('category')
logger.debug(
f'Feature {fet_name} loaded with {len(fd_df)} spikes, {fd_df.shape[1] - 1} dimensions '
)
images = []
titles = []
for cc in combinations(map(str, range(len(fd_df.columns) - 1)),
r=2):
fet_title = f'{fet_name}:{cc[1]} vs {fet_name}:{cc[0]}'
x_range = (np.percentile(fd_df[cc[0]], 0.01),
np.percentile(fd_df[cc[0]], 99.9))
y_range = (np.percentile(fd_df[cc[1]], 0.01),
np.percentile(fd_df[cc[1]], 99.9))
logger.debug(
f'shading {len(fd_df)} points in {fd_df.shape[1] - 1} dimensions'
)
canvas = ds.Canvas(plot_width=300,
plot_height=300,
x_range=x_range,
y_range=y_range)
agg = canvas.points(fd_df,
x=cc[0],
y=cc[1],
agg=ds.count_cat('clu_id'))
with np.errstate(invalid='ignore'):
img = ds_tf.shade(agg, how='log', color_key=color_keys)
img = img if cli_args.no_spread else ds_tf.spread(img,
px=1)
images.append(img)
titles.append(fet_title)
logger.debug(f'Creating plot for {fet_name}')
fet_fig = ds_plot_features(images, how='log', fet_titles=titles)
crf.write(fig2html(fet_fig) + '</br>\n')
plt.close(fet_fig)
def nodesplot(nodes, name=None, canvas=None, cat=None):
canvas = ds.Canvas(**cvsopts) if canvas is None else canvas
aggregator=None if cat is None else ds.count_cat(cat)
agg=canvas.points(nodes,'x','y',aggregator)
return tf.spread(tf.shade(agg, cmap=["#FF3333"]), px=3, name=name)
def test_spread():
p = 0x7d00007d
g = 0x7d00FF00
b = 0x7dFF0000
data = np.array([[p, p, 0, 0, 0], [p, g, 0, 0, 0], [0, 0, 0, 0, 0],
[0, 0, 0, b, 0], [0, 0, 0, 0, 0]],
dtype='uint32')
coords = [np.arange(5), np.arange(5)]
img = tf.Image(data, coords=coords, dims=dims)
s = tf.spread(img)
o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000,
0x7dff0000]])
np.testing.assert_equal(s.data, o)
assert (s.x_axis == img.x_axis).all()
assert (s.y_axis == img.y_axis).all()
assert s.dims == img.dims
s = tf.spread(img, px=2)
o = np.array([[0xed00863b, 0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000],
[0xed00863b, 0xed00863b, 0xf581411c, 0xdc904812, 0x7dff0000],
[0xed00863b, 0xf581411c, 0xed864419, 0xbca85600, 0x7dff0000],
[0xbc00a82a, 0xdc904812, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000,
0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, shape='square')
o = np.array([[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xed00863b, 0xed00863b, 0xbc00a82a, 0x00000000, 0x00000000],
[0xbc00a82a, 0xbc00a82a, 0xbca85600, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000,
0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, how='add')
o = np.array([[0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000],
[0xff007db7, 0xff007db7, 0xfa007f3e, 0x00000000, 0x00000000],
[0xfa007f3e, 0xfa007f3e, 0xfa7f7f00, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000, 0x7dff0000],
[0x00000000, 0x00000000, 0x7dff0000, 0x7dff0000,
0x7dff0000]])
np.testing.assert_equal(s.data, o)
mask = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
s = tf.spread(img, mask=mask)
o = np.array([[0xbc00a82a, 0xbc00007d, 0x7d00ff00, 0x00000000, 0x00000000],
[0xbc00007d, 0xbc00a82a, 0x7d00007d, 0x00000000, 0x00000000],
[0x7d00ff00, 0x7d00007d, 0xbca85600, 0x00000000, 0x7dff0000],
[0x00000000, 0x00000000, 0x00000000, 0x7dff0000, 0x00000000],
[0x00000000, 0x00000000, 0x7dff0000, 0x00000000,
0x7dff0000]])
np.testing.assert_equal(s.data, o)
s = tf.spread(img, px=0)
np.testing.assert_equal(s.data, img.data)
pytest.raises(ValueError, lambda: tf.spread(img, px=-1))
pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones(2)))
pytest.raises(ValueError, lambda: tf.spread(img, mask=np.ones((2, 2))))
layout = go.Layout(
xaxis = dict(title = 'Year Built'),
yaxis = dict(title = 'Average Number Of FLoors')
)
fig = go.Figure(data = [trace], layout = layout)
py.iplot(fig, filename = 'AvgNumOfFloor')
assess =ny[['AssessTot', 'AssessLand','lon','lat']].copy()
assess['AssBuilding'] = assess['AssessTot'].sub(assess['AssessLand'], axis=0) # Subtract to get Building Value
labels = ['A', 'B', 'C']
assess['AssLandBins'] = pd.qcut(assess.AssessLand, 3, labels=labels)
assess['AssBuldBins'] = pd.qcut(assess.AssBuilding, 3, labels=labels)
assess['Comb'] = assess.AssLandBins.astype(str) + '_' + assess.AssBuldBins.astype(str)
assess.Comb = assess.Comb.astype('category')
colors = {'A_A': '#e8a8e8', 'A_B': '#a4acac', 'A_C': '#c85a5a',
'B_A': '#b0a5df', 'B_B': '#ad9ea5', 'B_C': '#c85356',
'C_A': '#64ccbe', 'C_B': '#a27f8c', 'C_C': '#c74249'}
NewYorkCity = (( -74.29, -73.69), (40.49, 40.92))
cvs = ds.Canvas(700, 700, * NewYorkCity)
agg = cvs.points(assess, 'lon', 'lat', ds.count_cat('Comb'))
view = tf.shade(agg, color_key = colors)
export(tf.spread(view, px=1), 'Cloropleth')
def run_kk(params, run_kk=True):
cfg,maxc, target_path = params
tt_fname = target_path.name
tetrode_file_stem = tt_fname.split(".")[0]
tetrode_file_elecno = tt_fname.split(".")[-1]
working_dir = target_path.parent
logging.debug(f'Tetrode name: {tt_fname}, stem: {tetrode_file_stem}, ElecNo: {tetrode_file_elecno}')
clu_file = working_dir / (tetrode_file_stem + f'.clu.{tetrode_file_elecno}')
if clu_file.exists() and cfg['skip']:
logging.error(f'Clu file {clu_file} exists. Skipping.')
run_kk = False
# Read in feature validity
validity_path = target_path.with_suffix('.validity')
if not validity_path.exists():
logger.warning('No explicit feature validity given, falling back to default = all used.')
with open(validity_path) as vfp:
validity_string = vfp.readline()
logger.debug(f'Channel validity: {validity_string}')
# Combine executable and arguments
kk_executable = cfg["kk_executable"]
kk_cmd = f'{kk_executable} {tetrode_file_stem} -ElecNo {tetrode_file_elecno} -UseFeatures {validity_string} -MaxPossibleClusters {maxc}'
if cfg['KKv3']:
kk_cmd += ' -UseDistributional 0'
kk_cmd_list = kk_cmd.split(' ')
logger.debug(f'KK COMMAND: {kk_cmd}')
logger.debug(f'KK COMMAND LIST: {kk_cmd_list}')
# Call KlustaKwik and gather output
# TODO: Use communicate to interact with KK, i.e. write to log and monitor progress
# see https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
logger.info('Starting KlustaKwik process')
if cfg['PRINT_KK_OUTPUT']:
stdout = None
else:
stdout = subprocess.PIPE
if run_kk:
kk_call = subprocess.run(kk_cmd_list, stderr=subprocess.STDOUT, stdout=stdout)
kk_error = kk_call.returncode
logger.debug('Writing KlustaKwik log file')
logger.debug('Clu File: ' + str(clu_file))
if kk_call.stdout is not None:
with open(clu_file.with_suffix('.log'), 'w') as log_file:
log_file.write(kk_call.stdout.decode('ascii'))
else:
logging.warning('Missing stdout, not writing log file!')
# Check call return code and output
if kk_error:
logging.error(f'KlustaKwik error code: {kk_error}')
exit(kk_error)
else:
logging.debug(f'KlustaKwik successful: {kk_error}')
# Load clu file
logger.debug(f'Loading {clu_file}')
clu_df = pd.read_csv(clu_file, dtype='category', names=['cluster_id'], skiprows=1)
cluster_labels = clu_df['cluster_id'].cat.categories
num_clusters = len(cluster_labels)
logger.info(f'{len(clu_df)} spikes in {num_clusters} clusters')
# Find all feature .fd files
feature_files = list(working_dir.glob(tetrode_file_stem + '_*.fd'))
ff_sizes = [ff.stat().st_mtime for ff in feature_files]
feature_files = [f for t, f in sorted(zip(ff_sizes, feature_files))]
if not len(feature_files):
raise FileNotFoundError(f'No Feature Files found in {working_dir}')
# TODO: Stupid, the feature names are in the .fd file already
feature_names = [str(ff.name).split(tetrode_file_stem + '_')[1].split('.')[0] for ff in feature_files]
logger.info(f'Loading features: {feature_names}')
color_keys = cfg['CLUSTER_COLORS']
with open(clu_file.with_suffix('.html'), 'w') as crf:
crf.write('<head></head><body><h1>{}</h1>'.format(clu_file.name))
for fd_file, fet_name in zip(feature_files, feature_names):
crf.write('<h3>Feature: {}</h3>\n'.format(fet_name))
logger.info(f'Generating images for feature {fet_name}')
if not fd_file.exists():
continue
logger.debug(f'Loading {fd_file}')
mat_fet = h5s.loadmat(str(fd_file), appendmat=False)
fd_df = pd.DataFrame(mat_fet['FeatureData'])
fd_df.rename(columns={c: str(c) for c in fd_df.columns}, inplace=True)
if not len(clu_df) == len(fd_df):
raise ValueError(f'Number of cluster labels ({num_clusters}) does not match number of spikes'
f'in {fd_file} ({len(fd_df)})')
fd_df['clu_id'] = clu_df.cluster_id.astype('category')
logger.debug(f'Feature {fet_name} loaded with {len(fd_df)} spikes, {fd_df.shape[1] - 1} dimensions ')
images = []
titles = []
for cc in combinations(map(str, range(len(fd_df.columns) - 1)), r=2):
fet_title = f'x: {fet_name}:{cc[0]} vs y: {fet_name}:{cc[1]}'
x_range = (np.percentile(fd_df[cc[0]], 0.01), np.percentile(fd_df[cc[0]], 99.9))
y_range = (np.percentile(fd_df[cc[1]], 0.01), np.percentile(fd_df[cc[1]], 99.9))
logger.debug(f'shading {len(fd_df)} points in {fd_df.shape[1] - 1} dimensions')
canvas = ds.Canvas(plot_width=300, plot_height=300, x_range=x_range, y_range=y_range)
try:
agg = canvas.points(fd_df, x=cc[0], y=cc[1], agg=ds.count_cat('clu_id'))
with np.errstate(invalid='ignore'):
img = ds_tf.shade(agg, how='log', color_key=color_keys)
img = img if cfg['no_spread'] else ds_tf.spread(img, px=1)
except ZeroDivisionError:
img = None
images.append(img)
titles.append(fet_title)
logger.debug(f'Creating plot for {fet_name}')
fet_fig = ds_plot_features(images, how='log', fet_titles=titles)
crf.write(fig2html(fet_fig) + '</br>\n')
plt.close(fet_fig)
def test_float32_spread():
data = np.array(
[[1, 1, np.nan, np.nan, np.nan], [1, 2, np.nan, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan],
[np.nan, np.nan, np.nan, 3, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan]],
dtype='float32')
coords = [np.arange(5), np.arange(5)]
arr = xr.DataArray(data, coords=coords, dims=dims)
s = tf.spread(arr)
o = np.array([[5, 5, 3, np.nan, np.nan], [5, 5, 3, np.nan, np.nan],
[3, 3, 5, 3, 3], [np.nan, np.nan, 3, 3, 3],
[np.nan, np.nan, 3, 3, 3]])
np.testing.assert_equal(s.data, o)
assert (s.x_axis == arr.x_axis).all()
assert (s.y_axis == arr.y_axis).all()
assert s.dims == arr.dims
s = tf.spread(arr, px=2)
o = np.array([[5, 5, 5, 3, np.nan], [5, 5, 8, 6, 3], [5, 8, 7, 5, 3],
[3, 6, 5, 3, 3], [np.nan, 3, 3, 3, 3]])
np.testing.assert_equal(s.data, o)
s = tf.spread(arr, shape='square')
o = np.array([[5, 5, 3, np.nan, np.nan], [5, 5, 3, np.nan, np.nan],
[3, 3, 5, 3, 3], [np.nan, np.nan, 3, 3, 3],
[np.nan, np.nan, 3, 3, 3]])
np.testing.assert_equal(s.data, o)
s = tf.spread(arr, how='min')
o = np.array([[1, 1, 1, np.nan, np.nan], [1, 1, 1, np.nan, np.nan],
[1, 1, 2, 3, 3], [np.nan, np.nan, 3, 3, 3],
[np.nan, np.nan, 3, 3, 3]])
np.testing.assert_equal(s.data, o)
s = tf.spread(arr, how='max')
o = np.array([[2, 2, 2, np.nan, np.nan], [2, 2, 2, np.nan, np.nan],
[2, 2, 3, 3, 3], [np.nan, np.nan, 3, 3, 3],
[np.nan, np.nan, 3, 3, 3]])
np.testing.assert_equal(s.data, o)
mask = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
data = np.array([[np.nan, np.nan, np.nan, 1, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan],
[np.nan, 1, np.nan, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan]],
dtype='float32')
arr = xr.DataArray(data, coords=coords, dims=dims)
s = tf.spread(arr, mask=mask)
o = np.array([[0, 0, 0, 1, 0], [1, 0, 2, 0, 1], [0, 1, 0, 0, 0],
[1, 0, 1, 0, 0], [0, 0, 0, 0, 0]])
o = np.array([[np.nan, np.nan, np.nan, 1,
np.nan], [1, np.nan, 2, np.nan, 1],
[np.nan, 1, np.nan, np.nan, np.nan],
[1, np.nan, 1, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan]])
np.testing.assert_equal(s.data, o)
s = tf.spread(arr, px=0)
np.testing.assert_equal(s.data, arr.data)
pytest.raises(ValueError, lambda: tf.spread(arr, px=-1))
pytest.raises(ValueError, lambda: tf.spread(arr, mask=np.ones(2)))
pytest.raises(ValueError, lambda: tf.spread(arr, mask=np.ones((2, 2))))
trees['y_sp'].values)
trees = trees[(trees['lon'] < -60) & (trees['lon'] > -100) &
(trees['lat'] < 60) & (trees['lat'] > 20)]
#Also using project 2 code to get a map locating the trees.
#Defining some helper functions for DataShader
background = "black"
export = partial(export_image, background=background, export_path="export")
cm = partial(colormap_select, reverse=(background != "black"))
NewYorkCity = ((-74.29, -73.69), (40.49, 40.92))
cvs = ds.Canvas(700, 700, *NewYorkCity)
agg = cvs.points(trees, 'lon', 'lat')
view = tf.shade(agg, cmap=cm(viridis), how='log')
export(tf.spread(view, px=2), 'trees')
agg = cvs.points(trees, 'lon', 'lat', ds.mean('health'))
view = tf.shade(agg, cmap=cm(viridis), how='eq_hist')
export(tf.spread(view, px=2), 'trees_health')
#%%
"""
The block below is the actual Dash App. It simply displays the images with the
user inputs defined by Radio bottons. It then subsets the dataframe and
displays the appropreiate histogram.
"""
#%%
# https://www.youtube.com/watch?v=wv2MXJIdKRY
#https://dash.plot.ly/getting-started-part-2
app = dash.Dash()
def nodesplot(nodes, name=None, canvas=None, cat=None):
canvas = ds.Canvas(**cvsopts) if canvas is None else canvas
aggregator = None if cat is None else ds.count_cat(cat)
agg = canvas.points(nodes, 'x', 'y', aggregator)
return tf.spread(tf.shade(agg, cmap=["#FF3333"]), px=3, name=name)
# This shows us the distribution, but it's subject to some biases discussed in the Anaconda notebook [Plotting Perils](https://anaconda.org/jbednar/plotting_pitfalls/notebook).
#
# Here is what the same plot would look like in datashader:
#
#
# In[18]:
cvs = ds.Canvas(800,
500,
x_range=(ny['yearbuilt'].min(), ny['yearbuilt'].max()),
y_range=(ny['numfloors'].min(), ny['numfloors'].max()))
agg = cvs.points(ny, 'yearbuilt', 'numfloors')
view = tf.shade(agg, cmap=cm(Greys9), how='log')
export(tf.spread(view, px=2), 'yearvsnumfloors')
# That's technically just a scatterplot, but the points are smartly placed and colored to mimic what one gets in a heatmap. Based on the pixel size, it will either display individual points, or will color the points of denser regions.
#
# Datashader really shines when looking at geographic information. Here are the latitudes and longitudes of our dataset plotted out, giving us a map of the city colored by density of structures:
# In[10]:
NewYorkCity = ((-74.29, -73.69), (40.49, 40.92))
cvs = ds.Canvas(700, 700, *NewYorkCity)
agg = cvs.points(ny, 'lon', 'lat')
view = tf.shade(agg, cmap=cm(inferno), how='log')
export(tf.spread(view, px=2), 'firery')
# Interestingly, since we're looking at structures, the large buildings of Manhattan show up as less dense on the map. The densest areas measured by number of lots would be single or multi family townhomes.
#
def show_velphase(ds, ray_df, ray_start, ray_end, triray, filename):
# take in the yt dataset (ds) and a ray as a dataframe
# preliminaries
rs = ray_start.ndarray_view()
re = ray_end.ndarray_view()
imsize = 500
core_width = 10.
proper_box_size = ds.get_parameter(
'CosmologyComovingBoxSize') / ds.get_parameter(
'CosmologyHubbleConstantNow') * 1000. # in kpc
redshift = ds.get_parameter('CosmologyCurrentRedshift')
# take out a "core sample" that extends along the ray with a width given by core_width
ad = ds.r[rs[0]:re[0], rs[1] - 0.5 * core_width / proper_box_size:rs[1] +
0.5 * core_width / proper_box_size,
rs[2] - 0.5 * core_width / proper_box_size:rs[2] +
0.5 * core_width / proper_box_size]
cell_vol = ad["cell_volume"]
cell_mass = ad["cell_mass"]
cell_size = np.array(cell_vol)**(1. / 3.) * proper_box_size
x_cells = ad['x'].ndarray_view(
) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. )
y_cells = ad['y'].ndarray_view(
) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. )
z_cells = ad['z'].ndarray_view(
) * proper_box_size # + cell_size * (np.random.rand(np.size(cell_vol)) * 2. - 1. )
dens = np.log10(ad['density'].ndarray_view())
temp = np.log10(ad['temperature'].ndarray_view())
phase = np.chararray(np.size(temp), 4)
phase[temp < 19.] = 'hot'
phase[temp < 6.] = 'warm'
phase[temp < 5.] = 'cool'
phase[temp < 4.] = 'cold'
df = pd.DataFrame({
'x': x_cells,
'y': y_cells,
'z': z_cells,
'vx': ad["x-velocity"],
'vy': ad["y-velocity"],
'vz': ad["z-velocity"],
'temp': temp,
'dens': dens,
'phase': phase
})
df.phase = df.phase.astype('category')
cvs = dshader.Canvas(plot_width=imsize,
plot_height=imsize,
x_range=(np.min(df['x']), np.max(df['x'])),
y_range=(np.mean(df['y']) - 100. / 0.695,
np.mean(df['y']) + 100. / 0.695))
agg = cvs.points(df, 'x', 'y', dshader.count_cat('phase'))
img = tf.shade(agg, color_key=phase_color_key)
x_y = tf.spread(img, px=2, shape='square')
x_y.to_pil().save(filename + '_x_vs_y.png')
cvs = dshader.Canvas(plot_width=imsize,
plot_height=imsize,
x_range=(np.min(df['x']), np.max(df['x'])),
y_range=(-0.008, 0.008))
agg = cvs.points(df, 'x', 'vx', dshader.count_cat('phase'))
img = tf.shade(agg, color_key=phase_color_key)
x_vx = tf.spread(img, px=1, shape='square')
x_vx.to_pil().save(filename + '_x_vs_vx.png')
for species in species_dict.keys():
cvs = dshader.Canvas(plot_width=imsize,
plot_height=imsize,
x_range=(rs[0], re[0]),
y_range=(-0.008, 0.008))
vx = tf.shade(cvs.points(ray_df,
'x',
'x-velocity',
agg=reductions.mean(species_dict[species])),
how='eq_hist')
tf.set_background(vx, "white")
ray_vx = tf.spread(vx, px=4, shape='square')
pil = ray_vx.to_pil()
pil.save(filename + '_' + species + '_ray_vx.png', format='png')
