def save_unit_spike_trains(units, stimulus_list, c_add_unit_figures, c_add_retina_figure, by='angle'): print("Creating bar unit spike trains") if by == 'angle': get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(sorted, key=lambda e: e["stimulus"]["angle"]), ) nplots = get_nplots(stimulus_list,by) response = glia.apply_pipeline(get_solid,units) result = glia.plot_units(plot_spike_trains_by_angle,response, nplots=nplots, ncols=3,ax_xsize=10, ax_ysize=5, figure_title="Unit spike train by BAR angle") elif by == 'width': get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(sorted, key=lambda e: e["stimulus"]["width"]), ) nplots = get_nplots(stimulus_list,by) response = glia.apply_pipeline(get_solid,units) result = glia.plot_units(plot_spike_trains_by_trial,response, nplots=nplots, ncols=3,ax_xsize=10, ax_ysize=5, figure_title="Unit spike train by BAR angle") # nplots = len(speed_widths) c_add_unit_figures(result) glia.close_figs([fig for the_id,fig in result])
def plot_solid_versus_bar(fig, axis_gen, data, prepend, append): solids, bars_by_speed = data for speed in sorted(list(bars_by_speed.keys())): bars = bars_by_speed[speed] max_lifespan = max( bars, key=lambda e: e["stimulus"]["lifespan"])["stimulus"]["lifespan"] lifespans = set() for e in bars: # need to calculate duration of light over a particular point width = e["stimulus"]["width"] light_duration = int(np.ceil(width / speed)) lifespans.add(light_duration) light_wedge = glia.compose( partial(filter, lambda x: x["stimulus"]["lifespan"] in lifespans), partial(sorted, key=lambda x: x["stimulus"]["lifespan"])) sorted_solids = light_wedge(solids) sorted_bars = sorted(bars, key=lambda x: x["stimulus"]["width"]) xlim = glia.axis_continuation( lambda axis: axis.set_xlim(0, max_lifespan)) bar_text = glia.axis_continuation( partial(c_plot_bar, title="Bars with speed: {}".format(speed))) glia.plot_spike_trains(axis_gen, sorted_solids, prepend, append, continuation=xlim) glia.plot_spike_trains(axis_gen, sorted_bars, continuation=glia.compose(xlim, bar_text))
def save_acuity_chart(units, stimulus_list, c_unit_fig, c_add_retina_figure, prepend, append): "Compare SOLID light wedge to BAR response in corresponding ascending width." print("Creating acuity chart v3.") get_solids = glia.compose( glia.f_create_experiments(stimulus_list, prepend_start_time=prepend, append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), ) solids = glia.apply_pipeline(get_solids, units, progress=True) # offset to avoid diamond pixel artifacts get_bars_by_speed = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(sorted, key=lambda x: x["stimulus"]["angle"]), partial(sorted, key=lambda x: x["stimulus"]["width"]), partial(glia.group_by, key=lambda x: x["stimulus"]["speed"])) bars_by_speed = glia.apply_pipeline(get_bars_by_speed, units, progress=True) speeds = list(glia.get_unit(bars_by_speed)[1].keys()) for speed in sorted(speeds): print("Plotting acuity for speed {}".format(speed)) plot_function = partial(plot_acuity_v3, prepend=prepend, append=append, speed=speed) filename = "acuity-{}".format(speed) result = glia.plot_units( plot_function, partial(c_unit_fig, filename), solids, bars_by_speed, nplots=1, ncols=1, ax_xsize=5, ax_ysize=15, figure_title="Bars with speed {}".format(speed)) plot_function = partial(plot_dissimilarity, prepend=prepend, append=append, speed=speed) filename = "dissimilarity-{}".format(speed) result = glia.plot_units( plot_function, partial(c_unit_fig, filename), solids, bars_by_speed, nplots=1, ncols=1, ax_xsize=7, ax_ysize=7, figure_title="Dissimilarity matrix for bars with speed {}".format( speed))
def plot_motion_sensitivity(fig, axis_gen, data, nwidths, speeds, prepend, append): solids, bars_by_speed = data # We assume each band of widths is length 8 & first band is 2-16 # each subsequent band is twice the width # the list is sorted so the final bar has longest lifespan max_lifespan = bars_by_speed[speeds[0]][nwidths - 1]["stimulus"]["lifespan"] for i, speed in enumerate(speeds): base_width = 2**(i + 1) next_width = base_width widths = {base_width * (n + 1) for n in range(8)} bars_to_plot = list( filter(lambda x: x["stimulus"]["width"] in widths, bars_by_speed[speed])) bars_to_plot.sort(key=lambda x: x["stimulus"]["width"]) try: assert len(bars_to_plot) == 8 except: print("width", widths) print([bar["stimulus"] for bar in bars_to_plot]) raise xlim = glia.axis_continuation( lambda axis: axis.set_xlim(0, max_lifespan)) ylim = glia.axis_continuation(lambda axis: axis.set_ylim(0, 8)) bar_text = glia.axis_continuation( partial(c_plot_bar, title="Bars with speed: {} & base width: {}".format( speed, base_width))) glia.plot_spike_trains(axis_gen, bars_to_plot, continuation=glia.compose(xlim, bar_text, ylim)) if i == len(speeds) - 1: # After the last speed, Plot solid lifespans = set() for w in widths: light_duration = int(np.ceil(w / speed)) lifespans.add(light_duration) light_wedge = glia.compose( partial(filter, lambda x: x["stimulus"]["lifespan"] in lifespans), partial(sorted, key=lambda x: x["stimulus"]["lifespan"])) sorted_solids = light_wedge(solids) glia.plot_spike_trains(axis_gen, sorted_solids, prepend, append, continuation=xlim)
def save_acuity_direction(units, stimulus_list, c_unit_fig, c_add_retina_figure): "Make one direction plot per speed" get_direction = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(filter, lambda x: x["stimulus"]["barColor"] == "white"), partial(sorted, key=lambda e: e["stimulus"]["angle"]), partial(glia.group_by, key=lambda x: x["stimulus"]["speed"], value=lambda x: x)) response = glia.apply_pipeline(get_direction, units, progress=True) speeds = list(glia.get_unit(response)[1].keys()) nspeeds = len(speeds) for speed in sorted(speeds): print("Plotting DS for speed {}".format(speed)) plot_function = partial(plot_unit_response_for_speed, speed=speed) filename = "direction-{}".format(speed) glia.plot_units( plot_function, partial(c_unit_fig, filename), response, subplot_kw={"projection": "polar"}, ax_xsize=7, ax_ysize=7, figure_title="Units spike train for speed {}".format(speed), transpose=True)
def simulated_test(units, stimulus_list): # assert len(next(iter(units.values())).spike_train)==2200 test_pipeline = glia.compose(glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["GRATING"]), glia.f_group_by_stimulus(), glia.f_calculate_firing_rate_by_stimulus()) firing_rates = glia.apply_pipeline(test_pipeline, units, progress=True) for stimulus, rates in next(iter(firing_rates.values())).items(): for rate in rates: assert np.isclose(rate, 60, 1)
def get_fr_dsi_osi(units, stimulus_list): get_bar_firing_rate = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), glia.f_group_by_stimulus(), glia.f_calculate_firing_rate_by_stimulus(), ) bar_firing_rate = glia.apply_pipeline(get_bar_firing_rate, units, progress=True) get_bar_dsi = glia.compose(glia.by_speed_width_then_angle, glia.calculate_dsi_by_speed_width) bar_dsi = glia.apply_pipeline(get_bar_dsi, bar_firing_rate, progress=True) get_bar_osi = glia.compose(glia.by_speed_width_then_angle, glia.calculate_osi_by_speed_width) bar_osi = glia.apply_pipeline(get_bar_osi, bar_firing_rate, progress=True) return (bar_firing_rate, bar_dsi, bar_osi)
def save_unit_spike_trains(units, stimulus_list, c_unit_fig, c_add_retina_figure, prepend, append): print("Creating solid unit spike trains") get_solid = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=prepend,append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), ) response = glia.apply_pipeline(get_solid,units, progress=True) plot_function = partial(plot_spike_trains,prepend_start_time=prepend,append_lifespan=append) result = glia.plot_units(plot_function,response,ncols=1,ax_xsize=10, ax_ysize=5) c_unit_fig(result) glia.close_figs([fig for the_id,fig in result])
def save_integrity_chart(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating integrity chart") get_solid = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=1,append_lifespan=2), glia.f_has_stimulus_type(["SOLID"]), filter_lifespan ) response = glia.apply_pipeline(get_solid,units, progress=True) plot_function = partial(plot_spike_trains,prepend_start_time=1,append_lifespan=2) glia.plot_units(plot_function,c_unit_fig,response,ncols=1,ax_xsize=10, ax_ysize=5, figure_title="Integrity Test (5 Minute Spacing)")
def save_unit_spike_trains(units, stimulus_list, c_add_unit_figures, c_add_retina_figure, prepend, append): print("Creating solid unit spike trains") get_solid = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=prepend,append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), ) response = glia.apply_pipeline(get_solid,units) plot_function = partial(plot_spike_trains,prepend_start_time=prepend,append_lifespan=append) result = glia.plot_units(plot_function,response,ncols=1,ax_xsize=10, ax_ysize=5) c_add_unit_figures(result) glia.close_figs([fig for the_id,fig in result])
def save_unit_spike_trains(units, stimulus_list, c_add_unit_figures, c_add_retina_figure, by='angle'): print("Creating bar unit spike trains") if by == 'angle': get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(sorted, key=lambda e: e["stimulus"]["angle"]), ) nplots = get_nplots(stimulus_list, by) response = glia.apply_pipeline(get_solid, units, progress=True) result = glia.plot_units(plot_spike_trains_by_angle, response, nplots=nplots, ncols=3, ax_xsize=10, ax_ysize=5, figure_title="Unit spike train by BAR angle") elif by == 'width': get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), partial(sorted, key=lambda e: e["stimulus"]["width"]), ) nplots = get_nplots(stimulus_list, by) response = glia.apply_pipeline(get_solid, units, progress=True) result = glia.plot_units(plot_spike_trains_by_trial, response, nplots=nplots, ncols=3, ax_xsize=10, ax_ysize=5, figure_title="Unit spike train by BAR angle") # nplots = len(speed_widths) c_add_unit_figures(result) glia.close_figs([fig for the_id, fig in result])
def save_unit_psth(units, stimulus_list, c_add_unit_figures, c_add_retina_figure, prepend, append): print("Creating solid unit PSTH") get_psth = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=prepend,append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), glia.f_group_by_stimulus(), glia.concatenate_by_stimulus ) psth = glia.apply_pipeline(get_psth,units) plot_function = partial(plot_psth,prepend_start_time=prepend,append_lifespan=append) result = glia.plot_units(partial(plot_function,bin_width=0.01),psth,ax_xsize=10, ax_ysize=5) c_add_unit_figures(result) glia.close_figs([fig for the_id,fig in result])
def save_raster(units, stimulus_list, c_unit_fig, c_add_retina_figure, sort_by=glia.group_lifespan): print("Creating spike train raster plot") get_solid = glia.compose( glia.f_create_experiments(stimulus_list), partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, partial(sorted,key=sort_by) ) response = glia.apply_pipeline(get_solid,units, progress=True) glia.plot_units(glia.raster_group,c_unit_fig,response,nplots=1, ncols=1,ax_xsize=15, ax_ysize=10)
def save_unit_psth(units, stimulus_list, c_unit_fig, c_add_retina_figure, prepend, append): print("Creating solid unit PSTH") get_psth = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=prepend,append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), glia.f_group_by_stimulus(), glia.concatenate_by_stimulus ) psth = glia.apply_pipeline(get_psth,units, progress=True) plot_function = partial(plot_psth,prepend_start_time=prepend,append_lifespan=append) result = glia.plot_units(partial(plot_function,bin_width=0.01),psth,ax_xsize=10, ax_ysize=5) c_unit_fig(result) glia.close_figs([fig for the_id,fig in result])
def simulated_test(units, stimulus_list): assert len(next(iter(units.values())).spike_train)==2200 test_pipeline = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["GRATING"]), glia.f_group_by_stimulus(), glia.f_calculate_firing_rate_by_stimulus() ) firing_rates = glia.apply_pipeline(test_pipeline, units) for stimulus,rates in next(iter(firing_rates.values())).items(): for rate in rates: assert rate==1
def save_unit_wedges_v2(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating solid unit wedges") get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["SOLID","WAIT"]), partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, partial(sorted,key=lambda x: get_lifespan(x[1])) ) response = glia.apply_pipeline(get_solid,units, progress=True) glia.plot_units(plot_spike_train_triplet,c_unit_fig,response,nplots=1, ncols=1,ax_xsize=10, ax_ysize=5)
def get_fr_dsi_osi(units, stimulus_list): get_bar_firing_rate = glia.compose( glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["BAR"]), glia.f_group_by_stimulus(), glia.f_calculate_firing_rate_by_stimulus(), ) bar_firing_rate = glia.apply_pipeline(get_bar_firing_rate,units) get_bar_dsi = glia.compose( glia.by_speed_width_then_angle, glia.calculate_dsi_by_speed_width ) bar_dsi = glia.apply_pipeline(get_bar_dsi,bar_firing_rate) get_bar_osi = glia.compose( glia.by_speed_width_then_angle, glia.calculate_osi_by_speed_width ) bar_osi = glia.apply_pipeline(get_bar_osi,bar_firing_rate) return (bar_firing_rate, bar_dsi, bar_osi)
def save_unit_kinetics(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating solid unit kinetics") get_solid = glia.compose( glia.f_create_experiments(stimulus_list), partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, partial(sorted,key=lambda x: get_lifespan(x[2])) ) response = glia.apply_pipeline(get_solid,units, progress=True) # glia.plot_units(plot_group_spike_train,c_unit_fig,response,nplots=1, # ncols=1,ax_xsize=10, ax_ysize=5) glia.plot_units(glia.raster_group,c_unit_fig,response,nplots=1, ncols=1,ax_xsize=10, ax_ysize=5)
def save_unit_kinetics_v1(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating solid unit kinetics") for i in range(5): s = i*150 e = (i+1)*150 get_solid = glia.compose( glia.f_create_experiments(stimulus_list), lambda x: x[s:e], partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, partial(sorted,key=lambda x: get_lifespan(x[2])) ) response = glia.apply_pipeline(get_solid,units, progress=True) c = partial(c_unit_fig,"kinetics-{}".format(i)) glia.plot_units(glia.raster_group,c,response,nplots=1, ncols=1,ax_xsize=10, ax_ysize=5)
def save_unit_wedges(units, stimulus_list, c_unit_fig, c_add_retina_figure, prepend, append): print("Creating solid unit wedges") get_solid = glia.compose( glia.f_create_experiments(stimulus_list,prepend_start_time=prepend,append_lifespan=append), glia.f_has_stimulus_type(["SOLID"]), partial(sorted,key=lambda x: x["stimulus"]["lifespan"]) ) response = glia.apply_pipeline(get_solid,units, progress=True) colors = set() for solid in glia.get_unit(response)[1]: colors.add(solid["stimulus"]["backgroundColor"]) ncolors = len(colors) plot_function = partial(plot_spike_trains,prepend_start_time=prepend, append_lifespan=append) glia.plot_units(plot_function,c_unit_fig,response,nplots=ncolors, ncols=min(ncolors,5),ax_xsize=10, ax_ysize=5)
def save_integrity_chart_vFail(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating integrity chart") get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.filter_integrity, partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, integrity_fix_hack, partial(sorted,key=lambda x: x[0]["stimulus"]["stimulusIndex"]) ) response = glia.apply_pipeline(get_solid,units, progress=True) plot_function = partial(glia.raster_group) # c = partial(c_unit_fig,"kinetics-{}".format(i)) glia.plot_units(plot_function,c_unit_fig,response,ncols=1,ax_xsize=10, ax_ysize=5, figure_title="Integrity Test (5 Minute Spacing)") units_accuracy = glia.pmap(ideal_unit_classification_accuracy, response) c_add_retina_figure("integrity_accuracy",plot_units_accuracy(units_accuracy))
def filter_units_by_accuracy(units, stimulus_list, threshold=0.8): ntrial = len(list(filter( lambda x: 'metadata' in x['stimulus'] and "label" in x['stimulus']['metadata'] and \ x['stimulus']['metadata']['label']=='integrity', stimulus_list)))/3 ntrain = int(np.ceil(ntrial/2)) ntest = int(np.floor(ntrial/2)) tvt = glia.TVT(ntrain,ntest,0) get_solid = glia.compose( glia.f_create_experiments(stimulus_list), glia.filter_integrity, partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, glia.f_split_list(tvt) ) classification_data = glia.apply_pipeline(get_solid,units, progress=True) units_accuracy = glia.pmap(unit_classification_accuracy,classification_data) filter_threshold = glia.f_filter(lambda k,x: x['on']>threshold or x['off']>threshold) return set(filter_threshold(units_accuracy).keys())
def save_integrity_chart_v2(units, stimulus_list, c_unit_fig, c_add_retina_figure): print("Creating integrity chart") get_integrity= glia.compose( glia.f_create_experiments(stimulus_list), glia.filter_integrity, partial(glia.group_by, key=lambda x: x["stimulus"]["metadata"]["group"]), glia.group_dict_to_list, ) response = glia.apply_pipeline(get_integrity,units, progress=True) chronological = glia.apply_pipeline( partial(sorted,key=lambda x: x[0]["stimulus"]["stimulusIndex"]), response) plot_function = partial(glia.raster_group) # c = partial(c_unit_fig,"kinetics-{}".format(i)) glia.plot_units(plot_function,c_unit_fig,chronological,ncols=1,ax_xsize=10, ax_ysize=5, figure_title="Integrity Test (5 Minute Spacing)") ntrial = len(glia.get_unit(response)[1]) ntrain = int(np.ceil(ntrial/2)) ntest = int(np.floor(ntrial/2)) tvt = glia.TVT(ntrain,ntest,0) classification_data = glia.apply_pipeline( glia.f_split_list(tvt), response) units_accuracy = glia.pmap(unit_classification_accuracy,classification_data) plot_directory = os.path.join(config.plot_directory,"00-all") os.makedirs(plot_directory, exist_ok=True) with open(plot_directory + "/best_units.txt", "w") as f: sorted_units = sorted(units_accuracy.items(), key=lambda z: max(z[1]["off"],z[1]["on"]), reverse=True) for u in sorted_units: f.write(str(u)+"\n") c_add_retina_figure("integrity_accuracy",plot_units_accuracy(units_accuracy))
def save_unit_spike_trains(units, stimulus_list, c_unit_fig, c_add_retina_figure, width=None, height=None): print("Creating grating unit spike trains") get_solid = glia.compose(glia.f_create_experiments(stimulus_list), glia.f_has_stimulus_type(["GRATING"]), glia.f_split_by_wavelength()) response = glia.apply_pipeline(get_solid, units, progress=True) nplots = len(glia.get_unit(response)[1]) result = glia.plot_units( plot_spike_trains, response, nplots=nplots, ncols=3, ax_xsize=10, ax_ysize=5, figure_title="Unit spike train by GRATING waveperiod") c_unit_fig(result) glia.close_figs([fig for the_id, fig in result])
def save_images_h5(units, stimulus_list, name, frame_log, video_file, append): """Assumes each group is three stimuli with image in second position. Concatenate second stimuli with first 0.5s of third stimuli""" # open first so if there's a problem we don't waste time compression_level = 3 dset_filter = tables.filters.Filters(complevel=compression_level, complib='blosc:zstd') with tables.open_file(name + ".h5", 'w') as h5: class_resolver = get_classes_from_stimulus_list(stimulus_list) nclasses = len(class_resolver) frames, image_classes = glia.get_images_from_vid( stimulus_list, frame_log, video_file) image_class_num = list( map(lambda x: class_resolver[str(x)], image_classes)) idx_sorted_order = np.argsort(image_class_num) # save mapping of class_num target to class metadata # this way h5.root.image_classes[n] will give the class metadata string logger.info("create class_resolver with max string of 256") resolver = h5.create_carray(h5.root, "image_classes", tables.StringAtom(itemsize=256), (nclasses, )) img_class_array = np.array(image_classes, dtype="S256")[idx_sorted_order] for i, image_class in enumerate(img_class_array): resolver[i] = image_class atom = tables.Atom.from_dtype(frames[0].dtype) images = h5.create_carray(h5.root, "images", atom, (nclasses, *frames[0].shape), filters=dset_filter) frames = np.array(frames) nFrames = len(frames) for i, idx in enumerate(idx_sorted_order): if idx >= nFrames: logger.warn( f"skipping class {image_classes[idx]} as no accompanying frame. This should only occur if experiment stopped early." ) continue images[i] = frames[idx] print("finished saving images") get_image_responses = glia.compose( # returns a list partial(glia.create_experiments, stimulus_list=stimulus_list, progress=True, append_lifespan=append), partial(glia.group_by, key=lambda x: x["metadata"]["group"]), glia.group_dict_to_list, glia.f_filter(partial(glia.group_contains, "IMAGE")), # truncate to 0.5s glia.f_map(lambda x: [x[1], truncate(x[2], 0.5)]), glia.f_map(glia.merge_experiments), partial(glia.group_by, key=lambda x: x["metadata"]["cohort"]), # glia.f_map(f_flatten) ) image_responses = get_image_responses(units) ncohorts = len(image_responses) ex_cohort = glia.get_value(image_responses) images_per_cohort = len(ex_cohort) print("images_per_cohort", images_per_cohort) duration = ex_cohort[0]["lifespan"] d = int(np.ceil(duration * 1000)) # 1ms bins logger.info(f"ncohorts: {ncohorts}") # import pdb; pdb.set_trace() logger.info(f"nclasses: {nclasses}") if nclasses < 256: class_dtype = np.dtype('uint8') else: class_dtype = np.dtype('uint16') class_resolver_func = lambda c: class_resolver[str(c)] # determine shape experiments = glia.flatten_group_dict(image_responses) nE = len(experiments) d = int(np.ceil(duration * 1000)) # 1ms bins data_shape = (nE, d, Unit.nrow, Unit.ncol, Unit.nunit) print(f"writing to {name}.h5 with zstd compression...") data = h5.create_carray("/", "data", tables.Atom.from_dtype(np.dtype('uint8')), shape=data_shape, filters=dset_filter) target = h5.create_carray("/", "target", tables.Atom.from_dtype(class_dtype), shape=(nE, ), filters=dset_filter) glia.experiments_to_h5(experiments, data, target, partial(get_image_class_from_stim, class_resolver=class_resolver_func), append, class_dtype=class_dtype)
def save_letter_npz(units, stimulus_list, name, append): print( "Saving letter NPZ file. Warning: not including Off response--performance can be improved!" ) # TODO use merge_experiment # TODO add TEST!!! get_letters = glia.compose( partial(glia.create_experiments, stimulus_list=stimulus_list, progress=True, append_lifespan=append), partial(glia.group_by, key=lambda x: x["metadata"]["group"]), glia.group_dict_to_list, glia.f_filter(group_contains_letter), glia.f_map(lambda x: x[0:2]), partial(glia.group_by, key=lambda x: x[1]["size"]), glia.f_map( partial(glia.group_by, key=lambda x: x[1]["metadata"]["cohort"])), glia.f_map(glia.f_map(f_flatten)), glia.f_map(glia.f_map(balance_blanks))) letters = get_letters(units) sizes = sorted(list(letters.keys())) nsizes = len(sizes) ncohorts = len(list(letters.values())[0]) ex_letters = glia.get_value(list(letters.values())[0]) nletters = len(ex_letters) print("nletters", nletters) duration = ex_letters[0]["lifespan"] d = int(np.ceil(duration * 1000)) # 1ms bins nunits = len(units.keys()) tvt = glia.tvt_by_percentage(ncohorts, 60, 40, 0) logger.info(f"{tvt}, ncohorts: {ncohorts}") experiments_per_cohort = 11 training_data = np.full((nsizes, tvt.training * experiments_per_cohort, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') training_target = np.full((nsizes, tvt.training * experiments_per_cohort), 0, dtype='int8') validation_data = np.full((nsizes, tvt.validation * experiments_per_cohort, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') validation_target = np.full( (nsizes, tvt.validation * experiments_per_cohort), 0, dtype='int8') size_map = {s: i for i, s in enumerate(sizes)} for size, cohorts in letters.items(): X = glia.f_split_dict(tvt)(cohorts) logger.info(f"ncohorts: {len(cohorts)}") td, tt = glia.experiments_to_ndarrays(glia.training_cohorts(X), letter_class, append) logger.info(td.shape) missing_duration = d - td.shape[1] pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') size_index = size_map[size] training_data[size_index] = pad_td training_target[size_index] = tt td, tt = glia.experiments_to_ndarrays(glia.validation_cohorts(X), letter_class, append) pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') validation_data[size_index] = pad_td validation_target[size_index] = tt np.savez(name, training_data=training_data, training_target=training_target, validation_data=validation_data, validation_target=validation_target)
def compose_test(): test = glia.compose(lambda x: 3*x, lambda y: y**2, lambda z: z+1) assert test(4) == 145
def save_eyechart_npz(units, stimulus_list, name, append=0.5): print("Saving eyechart NPZ file.") # TODO add blanks get_letters = glia.compose( partial(glia.create_experiments, stimulus_list=stimulus_list, append_lifespan=append), partial(glia.group_by, key=lambda x: x["metadata"]["group"]), glia.group_dict_to_list, glia.f_filter(group_contains_letter), glia.f_map(lambda x: adjust_lifespan(x[1])), partial(glia.group_by, key=lambda x: x["size"]), glia.f_map( partial(glia.group_by, key=lambda x: x[1]["stimulus"]["metadata"]["cohort"])), glia.f_map(glia.f_map(f_flatten)), ) letters = get_letters(units) sizes = sorted(list(letters.keys())) nsizes = len(sizes) ncohorts = len(list(letters.values())[0]) ex_letters = glia.get_a_value(list(letters.values())[0]) nletters = len(ex_letters) print("nletters", nletters) duration = ex_letters[0]["lifespan"] d = int(np.ceil(duration * 1000)) # 1ms bins nunits = len(units.keys()) tvt = glia.tvt_by_percentage(ncohorts, 60, 40, 0) logger.info(f"{tvt}, {ncohorts}") training_data = np.full((nsizes, tvt.training, d, nunits), 0, dtype='int8') training_target = np.full((nsizes, tvt.training), 0, dtype='int8') validation_data = np.full((nsizes, tvt.validation, d, nunits), 0, dtype='int8') validation_target = np.full((nsizes, tvt.validation), 0, dtype='int8') test_data = np.full((nsizes, tvt.test, d, nunits), 0, dtype='int8') test_target = np.full((nsizes, tvt.test), 0, dtype='int8') size_map = {s: i for i, s in enumerate(sizes)} for size, experiments in letters.items(): split = glia.f_split_dict(tvt) flatten_cohort = glia.compose(glia.group_dict_to_list, f_flatten) X = glia.tvt_map(split(experiments), flatten_cohort) td, tt = glia.experiments_to_ndarrays(X.training, letter_class, append) size_index = size_map[size] training_data[size_index] = td training_target[size_index] = tt td, tt = glia.experiments_to_ndarrays(X.validation, letter_class, append) validation_data[size_index] = td validation_target[size_index] = tt td, tt = glia.experiments_to_ndarrays(X.test, letter_class, append) test_data[size_index] = td test_target[size_index] = tt np.savez(name, training_data=training_data, training_target=training_target, validation_data=validation_data, validation_target=validation_target)
def plot_solid_versus_bar_for_speed(fig, axis_gen, data, prepend, append, speed): # also assumes 5 contrasts logger.debug("plot solid versus bar for speed") solids, bars_by_speed = data bars = bars_by_speed[speed] max_lifespan = max( bars, key=lambda e: e["stimulus"]["lifespan"])["stimulus"]["lifespan"] lifespans = set() widths = set() colors = set() for e in bars: width = e["stimulus"]["width"] widths.add(width) color = e["stimulus"]["barColor"] # need to calculate duration of light over a particular point light_duration = int(np.ceil(width / speed)) lifespans.add(light_duration) colors.add(color) logger.debug("lifespans {}, widths {}".format(len(lifespans), len(widths))) # WARNING # assert len(lifespans)==len(widths) # keep charts aligned by row bar_ymap = {w: i for i, w in enumerate(sorted(list(widths)))} solid_ymap = {l: i for i, l in enumerate(sorted(list(lifespans)))} # used to map stimulus to proper row c_bar_ymap = lambda s: bar_ymap[s["width"]] c_solid_ymap = lambda s: solid_ymap[s["lifespan"]] sorted_colors = sorted(list(colors), reverse=True) ntrials = len(lifespans) xlim = glia.axis_continuation(lambda axis: axis.set_xlim(0, max_lifespan)) ylim = glia.axis_continuation(lambda axis: axis.set_ylim(0, ntrials)) color_text = lambda x: glia.axis_continuation( partial(c_plot_bar, title="Color: {}".format(x))) solid_continuation = lambda x: glia.compose(ylim, xlim, color_text(x)) bar_text = glia.axis_continuation(partial(c_plot_bar, title="Bar")) bar_continuation = glia.compose(ylim, xlim, bar_text) # we plot the solid first so they are all on the same row for color in sorted_colors: logger.debug('plotting SOLID for {}'.format(color)) light_wedge = glia.compose( partial(filter, lambda x: x["stimulus"]["lifespan"] in lifespans), partial(filter, lambda x: x["stimulus"]["backgroundColor"] == color), partial(sorted, key=lambda x: x["stimulus"]["lifespan"])) sorted_solids = light_wedge(solids) logger.debug( 'Solid lifespans are: ' + ",".join([str(s["stimulus"]["lifespan"]) for s in sorted_solids])) glia.plot_spike_trains(axis_gen, sorted_solids, prepend, append, continuation=solid_continuation(color), ymap=c_solid_ymap) for color in sorted_colors: filtered_bars = glia.compose( partial(filter, lambda x: x["stimulus"]["barColor"] == color), partial(sorted, key=lambda x: x["stimulus"]["width"])) sorted_bars = filtered_bars(bars) logger.debug('plotting BAR for {}'.format(color)) logger.debug( 'Bar widths are: ' + ",".join([str(s["stimulus"]["width"]) for s in sorted_bars])) glia.plot_spike_trains(axis_gen, sorted_bars, continuation=bar_continuation, ymap=c_bar_ymap)
from warnings import warn import sklearn.metrics as metrics from sklearn import manifold from sklearn.metrics import euclidean_distances import itertools import elephant from math import isclose from neo.core import SpikeTrain import quantities def distance_by_row(similarity): return list(map(lambda x: np.std(x), similarity)) def truncate_experiment(max_time, experiment): new = experiment.copy() train = experiment["spikes"] new["spikes"] = train[np.where(train < max_time)] return new # i = glia.compose( # glia.f_create_experiments(stimulus_list), # glia.f_has_stimulus_type(["SOLID"]), # partial(filter,lambda x: isclose(x["stimulus"]["lifespan"],0.5)), # lambda x: list(x), # ) icentroid = glia.compose(distance_by_row, np.min)
def save_acuity_image_npz(units, stimulus_list, name, append): "Assumes metadata includes a parameter to group by, as well as a blank image" get_letters = glia.compose( partial(glia.create_experiments, stimulus_list=stimulus_list, progress=True, append_lifespan=append), partial(glia.group_by, key=lambda x: x["metadata"]["group"]), glia.group_dict_to_list, glia.f_filter(partial(glia.group_contains, "IMAGE")), glia.f_map(lambda x: x[0:2]), partial(glia.group_by, key=lambda x: x[1]["metadata"]["parameter"]), glia.f_map( partial(glia.group_by, key=lambda x: x[1]["metadata"]["cohort"])), glia.f_map(glia.f_map(f_flatten)), glia.f_map(glia.f_map(partial(balance_blanks, key='image')))) letters = get_letters(units) sizes = sorted(list(letters.keys())) nsizes = len(sizes) ncohorts = len(list(letters.values())[0]) ex_letters = glia.get_value(list(letters.values())[0]) nletters = len(ex_letters) print("nletters", nletters) duration = ex_letters[0]["lifespan"] # small hack to fix bug in letters 0.2.0 letter_duration = ex_letters[1]['lifespan'] if duration != letter_duration: new_letters = {} for size, cohorts in letters.items(): new_letters[size] = {} for cohort, stimuli in cohorts.items(): new_letters[size][cohort] = list( map(lambda s: truncate(s, letter_duration), stimuli)) letters = new_letters d = int(np.ceil(duration * 1000)) # 1ms bins nunits = len(units.keys()) tvt = glia.tvt_by_percentage(ncohorts, 60, 40, 0) logger.info(f"{tvt}, ncohorts: {ncohorts}") experiments_per_cohort = 11 training_data = np.full((nsizes, tvt.training * experiments_per_cohort, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') training_target = np.full((nsizes, tvt.training * experiments_per_cohort), 0, dtype='int8') validation_data = np.full((nsizes, tvt.validation * experiments_per_cohort, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') validation_target = np.full( (nsizes, tvt.validation * experiments_per_cohort), 0, dtype='int8') size_map = {s: i for i, s in enumerate(sizes)} for size, cohorts in letters.items(): X = glia.f_split_dict(tvt)(cohorts) logger.info(f"ncohorts: {len(cohorts)}") td, tt = glia.experiments_to_ndarrays(glia.training_cohorts(X), acuity_image_class, append) logger.info(td.shape) missing_duration = d - td.shape[1] pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') size_index = size_map[size] training_data[size_index] = pad_td training_target[size_index] = tt td, tt = glia.experiments_to_ndarrays(glia.validation_cohorts(X), acuity_image_class, append) pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') validation_data[size_index] = pad_td validation_target[size_index] = tt np.savez(name, training_data=training_data, training_target=training_target, validation_data=validation_data, validation_target=validation_target)
def save_checkerboard_flicker_npz(units, stimulus_list, name, append, group_by, quad=False): "Psychophysics discrimination checkerboard 0.2.0" print("Saving checkerboard NPZ file.") get_checkers = glia.compose( partial( glia.create_experiments, progress=True, append_lifespan=append, # stimulus_list=stimulus_list,append_lifespan=0.5), stimulus_list=stimulus_list), partial(glia.group_by, key=lambda x: x["metadata"]["group"]), glia.group_dict_to_list, glia.f_filter(group_contains_checkerboard), glia.f_map( glia.f_filter(lambda x: x['stimulusType'] == 'CHECKERBOARD')), glia.f_map(glia.merge_experiments), partial(glia.group_by, key=group_by), glia.f_map(partial(glia.group_by, key=lambda x: x["size"])), glia.f_map( glia.f_map( partial(glia.group_by, key=lambda x: x["metadata"]["cohort"])))) checkers = get_checkers(units) max_duration = 0.0 for condition, sizes in checkers.items(): for size, cohorts in sizes.items(): for cohort, experiments in cohorts.items(): max_duration = max(max_duration, experiments[0]['lifespan']) max_duration += append print(f"max_duration: {max_duration}") conditions = sorted(list(checkers.keys())) print("Conditions:", name, conditions) nconditions = len(conditions) example_condition = glia.get_value(checkers) sizes = sorted(list(example_condition.keys())) nsizes = len(sizes) # TODO remove if max_duration < 9: print(example_condition) example_size = glia.get_value(example_condition) ncohorts = len(example_size) # print(list(checkers.values())) d = int(np.ceil(max_duration * 1000)) # 1ms bins tvt = glia.tvt_by_percentage(ncohorts, 60, 40, 0) logger.info(f"{tvt}, {ncohorts}") # (TODO?) 2 dims for first checkerboard and second checkerboard # 4 per cohort if quad: ntraining = tvt.training * 4 nvalid = tvt.validation * 4 else: ntraining = tvt.training * 2 nvalid = tvt.validation * 2 training_data = np.full( (nconditions, nsizes, ntraining, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') training_target = np.full((nconditions, nsizes, ntraining), 0, dtype='int8') validation_data = np.full( (nconditions, nsizes, nvalid, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') validation_target = np.full((nconditions, nsizes, nvalid), 0, dtype='int8') # test_data = np.full((nsizes,tvt.test,d,nunits),0,dtype='int8') # test_target = np.full((nsizes,tvt.test),0,dtype='int8') if quad: get_class = get_checker_quad_discrimination_class else: get_class = get_checker_discrimination_class condition_map = {c: i for i, c in enumerate(conditions)} size_map = {s: i for i, s in enumerate(sizes)} for condition, sizes in checkers.items(): for size, cohorts in sizes.items(): X = glia.f_split_dict(tvt)(cohorts) td, tt = glia.experiments_to_ndarrays(glia.training_cohorts(X), get_class, append) logger.info(td.shape) missing_duration = d - td.shape[1] pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') condition_index = condition_map[condition] size_index = size_map[size] training_data[condition_index, size_index] = pad_td training_target[condition_index, size_index] = tt td, tt = glia.experiments_to_ndarrays(glia.validation_cohorts(X), get_class, append) pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') validation_data[condition_index, size_index] = pad_td validation_target[condition_index, size_index] = tt print('saving to ', name) np.savez(name, training_data=training_data, training_target=training_target, validation_data=validation_data, validation_target=validation_target)
def compose_test(): test = glia.compose(lambda x: 3 * x, lambda y: y**2, lambda z: z + 1) assert test(4) == 145
def save_grating_npz(units, stimulus_list, name, append, group_by, sinusoid=False): "Psychophysics discrimination grating 0.2.0" print("Saving grating NPZ file.") if sinusoid: stimulus_type = "SINUSOIDAL_GRATING" else: stimulus_type = 'GRATING' get_gratings = glia.compose( partial(glia.create_experiments, stimulus_list=stimulus_list, append_lifespan=append), glia.f_filter(lambda x: x['stimulusType'] == stimulus_type), partial(glia.group_by, key=group_by), glia.f_map(partial(glia.group_by, key=lambda x: x["width"])), glia.f_map( glia.f_map( partial(glia.group_by, key=lambda x: x["metadata"]["cohort"])))) gratings = get_gratings(units) max_duration = 0.0 for condition, sizes in gratings.items(): for size, cohorts in sizes.items(): for cohort, experiments in cohorts.items(): max_duration = max(max_duration, experiments[0]['lifespan']) max_duration += append conditions = sorted(list(gratings.keys())) print("Conditions:", name, conditions) nconditions = len(conditions) example_condition = glia.get_value(gratings) sizes = sorted(list(example_condition.keys())) print("Sizes:", sizes) nsizes = len(sizes) example_size = glia.get_value(example_condition) ncohorts = len(example_size) # print(list(gratings.values())) d = int(np.ceil(max_duration * 1000)) # 1ms bins tvt = glia.tvt_by_percentage(ncohorts, 60, 40, 0) # 2 per cohort training_data = np.full((nconditions, nsizes, tvt.training * 2, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') training_target = np.full((nconditions, nsizes, tvt.training * 2), 0, dtype='int8') validation_data = np.full((nconditions, nsizes, tvt.validation * 2, d, Unit.nrow, Unit.ncol, Unit.nunit), 0, dtype='int8') validation_target = np.full((nconditions, nsizes, tvt.validation * 2), 0, dtype='int8') condition_map = {c: i for i, c in enumerate(conditions)} size_map = {s: i for i, s in enumerate(sizes)} for condition, sizes in gratings.items(): for size, cohorts in sizes.items(): X = glia.f_split_dict(tvt)(cohorts) td, tt = glia.experiments_to_ndarrays(glia.training_cohorts(X), get_grating_class_from_stim, append) missing_duration = d - td.shape[1] pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') condition_index = condition_map[condition] size_index = size_map[size] training_data[condition_index, size_index] = pad_td training_target[condition_index, size_index] = tt td, tt = glia.experiments_to_ndarrays(glia.validation_cohorts(X), get_grating_class_from_stim, append) pad_td = np.pad(td, ((0, 0), (0, missing_duration), (0, 0), (0, 0), (0, 0)), mode='constant') validation_data[condition_index, size_index] = pad_td validation_target[condition_index, size_index] = tt print('saving to ', name) np.savez(name, training_data=training_data, training_target=training_target, validation_data=validation_data, validation_target=validation_target)
def plot_acuity_v3(fig, axis_gen, data, prepend, append, speed): logger.debug("plot solid versus bar for speed") ax = next(axis_gen) solids, bars_by_speed = data bars = bars_by_speed[speed] max_lifespan = max( bars, key=lambda e: e["stimulus"]["lifespan"])["stimulus"]["lifespan"] lifespans = set() widths = set() angles = set() for e in bars: width = e["stimulus"]["width"] angle = e["stimulus"]["angle"] widths.add(width) angles.add(angle) # need to calculate duration of light over a particular point light_duration = int(np.ceil(width / speed)) lifespans.add(light_duration) logger.debug("lifespans {}, widths {}".format(len(lifespans), len(widths))) # WARNING # assert len(lifespans)==len(widths) # keep charts aligned by row angle_width = [] for w in sorted(widths): for a in sorted(angles): angle_width.append((a, w)) bar_ymap = {aw: i for i, aw in enumerate(angle_width)} # used to map stimulus to proper row c_bar_ymap = lambda s: bar_ymap[(s["angle"], s["width"])] nangles = len(angles) nwidths = len(widths) ny = nangles * nwidths xlim = glia.axis_continuation(lambda axis: axis.set_xlim(0, max_lifespan)) ylim = glia.axis_continuation(lambda axis: axis.set_ylim(0, ny)) bar_text = glia.axis_continuation( partial(c_plot_bar, title="{} angles x {} widths at {} px/s".format( nangles, nwidths, speed))) bar_continuation = glia.compose(ylim, xlim, bar_text) for e in bars: spikes = e["spikes"] lifespan = e["stimulus"]["lifespan"] angle = e["stimulus"]["angle"] width = e["stimulus"]["width"] y = bar_ymap[(angle, width)] glia.draw_spikes(ax, spikes, y + 0.2, y + 0.8) ax.fill([0, lifespan, lifespan, 0], [y, y, y + 1, y + 1], facecolor="gray", edgecolor="none", alpha=0.1) ax.yaxis.set_ticks(np.arange(0, nwidths * nangles, nangles)) ax.yaxis.set_ticklabels(sorted(list(widths))) ax.set_ylabel("Bar Width in pixels (angle changes each row)") ax.set_xlabel("Time in seconds")