def __init__(self, data: DataFrame, stimulus: dict, sample_rate: int) -> None: self.initialized = False # type: bool self.converted = False DataFrame.__init__(self, data.values, data.axes) Stimulus.__init__(self, stimulus) self.sample_rate = sample_rate
def test_enumerate(): array1 = DataFrame(np.arange(24).reshape(2, 3, 4), [np.arange(2), np.arange(3), np.arange(4)]) result1 = list(array1.enumerate()) correct1 = [[[0, 0, 0], 0], [[0, 0, 1], 1], [[0, 0, 2], 2], [[0, 0, 3], 3], [[0, 1, 0], 4], [[0, 1, 1], 5], [[0, 1, 2], 6], [[0, 1, 3], 7], [[0, 2, 0], 8], [[0, 2, 1], 9], [[0, 2, 2], 10], [[0, 2, 3], 11], [[1, 0, 0], 12], [[1, 0, 1], 13], [[1, 0, 2], 14], [[1, 0, 3], 15], [[1, 1, 0], 16], [[1, 1, 1], 17], [[1, 1, 2], 18], [[1, 1, 3], 19], [[1, 2, 0], 20], [[1, 2, 1], 21], [[1, 2, 2], 22], [[1, 2, 3], 23]] assert(np.array_equal(result1, correct1))
def test_group_mean(): data = DataFrame(np.arange(81).reshape((3, 3, 3, 3)), [np.arange(3), np.arange(3), np.arange(1, 4), np.arange(2, 5)]) group1 = [np.array(['a', 'b', 'a']), np.array(['c', 'd', 'c'])] correct1 = np.array([[[[12, 13, 14], [12, 13, 14]], [[12, 13, 14], [12, 13, 14]]], [[[39, 40, 41], [39, 40, 41]], [[39, 40, 41], [39, 40, 41]]], [[[66, 67, 68], [66, 67, 68]], [[66, 67, 68], [66, 67, 68]]]]) axes1 = [np.arange(3), np.array(['a', 'b']), np.array(['c', 'd']), np.arange(2, 5)] assert data.group_by(group1, lambda x: np.mean(x, axis=1)) == DataFrame(correct1, axes1) group2 = [np.array(['a', 'b', 'a'])] correct2 = np.array([[[[9, 10, 11], [12, 13, 14], [15, 16, 17]], [[9, 10, 11], [12, 13, 14], [15, 16, 17]]], [[[36, 37, 38], [39, 40, 41], [42, 43, 44]], [[36, 37, 38], [39, 40, 41], [42, 43, 44]]], [[[63, 64, 65], [66, 67, 68], [69, 70, 71]], [[63, 64, 65], [66, 67, 68], [69, 70, 71]]]]) axes2 = [np.arange(3), np.array(['a', 'b']), np.arange(1, 4), np.arange(2, 5)] result2 = data.group_by(group2, lambda x: np.mean(x, axis=1)) assert result2 == DataFrame(correct2, axes2) correct2_5 = np.array([[[12, 13, 14], [12, 13, 14]], [[39, 40, 41], [39, 40, 41]], [[66, 67, 68], [66, 67, 68]]]) axes2_5 = [np.arange(3), np.array(['a', 'b']), np.arange(2, 5)] result2_5 = data.group_by(group2, lambda x: np.mean(x, axis=(1, 2))) assert result2_5 == DataFrame(correct2_5, axes2_5)
def full_trace(record_file: File, ax: Axes): activity = DataFrame.load(record_file["spike"]) lever = load_mat(record_file['response']) new_lever = resample(lever.values[0], lever.sample_rate, record_file.attrs['frame_rate']) for x in range(activity.shape[0]): ax.plot(_scale(activity.values[x, :] / 5)) ax.plot(_scale(new_lever) / 2 - 3, color=COLORS[0])
def visualize_prediction(data_file: File, params: MotionParams): """Show 3 PCs of neuron activity and the plane in linear SVC classifier.""" lever = get_trials(data_file, params) trials = fold_by(DataFrame.load(data_file['spike']), lever, data_file.attrs['frame_rate'], True) k_cluster = k_means(lever.values, 2)[1].astype(np.bool) main_cluster, null_cluster = np.flatnonzero( k_cluster)[:20], np.flatnonzero(~k_cluster)[:10] all_neurons = np.hstack([ trials.values.take(main_cluster, 1), trials.values.take(null_cluster, 1) ]) all_results = np.array([0] * 20 + [1] * 10) training_X = all_neurons.swapaxes(0, 1).reshape(all_neurons.shape[1], -1) pca_weigths = PCA(20).fit_transform(training_X) classifier = SVC(kernel='linear') classifier.fit(pca_weigths, all_results) classifier.score(pca_weigths, all_results) coef = classifier.coef_[0] intercept = classifier.intercept_ xx, yy = np.meshgrid( np.linspace(pca_weigths[:, 0].min(), pca_weigths[:, 0].max(), 20), np.linspace(pca_weigths[:, 1].min(), pca_weigths[:, 1].max(), 20)) z = (-intercept - xx * coef[0] - yy * coef[1]) / coef[2] with Figure(projection='3d') as ax: ax[0].scatter(*pca_weigths[:, 6:9].T, color=np.asarray(COLORS)[all_results], s=50) ax[0].plot_surface(xx, yy, z, alpha=0.4, color=COLORS[-1]) ax[0].set_zlim(pca_weigths[:, 2].min(), pca_weigths[:, 2].max())
def svr_parameters(data_file: File, info: Dict[str, str]): lever = load_mat(data_file['response']) values = devibrate(lever.values[0], sample_rate=lever.sample_rate) y = InterpolatedUnivariateSpline(lever.axes[0], values)(data_file['spike']['y'])[1:] X = data_file['spike']['data'][:, 1:] gammas = np.linspace(-8, -5, 12, endpoint=False) Cs = np.linspace(3, 15, 12, endpoint=False) def pred(gamma, C): hat = cross_predict(X, y, svr.predictor_factory(y, gamma=10**gamma, C=C, epsilon=1E-3), section_mi=False) return mutual_info(y, hat) res = map_table(pred, gammas, Cs) save_path = join(res_folder, f"svr_params_test_{info['id']}_{info['session']}.npz") np.savez_compressed(save_path, values=np.asarray(res), axes=[gammas, Cs]) res_df = DataFrame(np.asarray(res), [gammas, Cs]) with Figure() as (ax, ): labeled_heatmap(ax, res_df.values, res_df.axes[1], res_df.axes[0]) print('done')
def draw_noise(data_files: Dict[int, File], neuron_id: int, params: MotionParams): last_day = max(data_files.keys()) lever = load_mat(data_files[last_day]['response']) neuron_rate = data_files[last_day].attrs['frame_rate'] neurons = common_axis([DataFrame.load(x['spike']) for x in data_files.values()]) good, bad, anti = classify_cells(motion_corr( lever, neurons[-1], neuron_rate, 16000, params), 0.001) amp = list() corrs: Dict[str, List[List[float]]] = {'good': [], 'unrelated': [], 'between': []} for (day_id, data_file), neuron in zip(data_files.items(), neurons): if day_id == last_day: continue lever = load_mat(data_file['response']) corrs['good'].append(_take_triu(noise_autocorrelation(lever, neuron[good], neuron_rate))) corrs['unrelated'].append(_take_triu(noise_autocorrelation(lever, neuron[bad | anti], neuron_rate))) corrs['between'].append(_take_triu(noise_correlation(lever, neuron[good], neuron[bad | anti], neuron_rate))) lever.center_on("motion", **params) neuron_trials = fold_by(neuron, lever, neuron_rate, True) amp.append(neuron_trials.values[np.argwhere(neuron.axes[0] == neuron_id)[0, 0], :, :].max(axis=1)) with Figure(join(project_folder, 'report', 'img', f'noise_corr_{neuron_id}.svg')) as (ax,): day_ids = [x for x in data_files.keys() if x != last_day] for idx, (group_str, group) in enumerate(corrs.items()): ax.errorbar(day_ids, [np.mean(x) for x in group], yerr=[_sem(x) for x in group], color=COLORS[idx], label=group_str) ax2 = ax.twinx() ax2.errorbar(day_ids, [np.mean(x) for x in amp], [_sem(x) for x in amp], color=COLORS[-1]) ax.set_title(str(neuron_id)) ax.legend()
def load_plain(file_path: Union[str, Path]) -> SparseRec: raw_log = unpack(file_path) mode_switches = flatten_events(raw_log['event'], ('passive', 'wait_push', 'push', 'pull')) trial_switches = flatten_events(raw_log['event'], ('trial', 'reward', 'intertrial')) trials = events2trials(trial_switches, mode_switches) sample_rate = 1000 config = { **raw_log["design"], **raw_log["hardware"], "blank_time": raw_log["design"]["program"]["violation"] / 1E6, "stim_time": raw_log["design"]["program"]["trial"] / 1E6 } stimulus = { 'config': config, "timestamps": trials["trial"], "rewardstamps": trials["reward"], "sequence": { "trials": trials } } trace = DataFrame( 32768 - raw_log['lever']['value'][np.newaxis, :], [np.array(["right-lever"]), raw_log['lever']['timestamp']]) return SparseRec(trace, stimulus, sample_rate)
def test_find_deviate(lever_file): raw_movement = lever_file['response']['mvmtdata'].ravel() calibration_factor = lever_file['params']['lev_cal'] trace = raw_movement / calibration_factor recording = DataFrame(trace, [np.arange(len(trace))]) find_deviate(recording) assert (False)
def make_sample_neurons(spike_framerate: Tuple[Dict[str, np.ndarray], float], log: SparseRec, params: Dict[str, float]) -> SparseRec: lever = log.center_on("motion", **params).fold_trials() lever.values = np.squeeze(lever.values, 0) lever.axes = lever.axes[1:] filtered = devibrate_rec(lever, params) spikes, frame_rate = spike_framerate return fold_by(DataFrame.load(spikes), filtered, frame_rate, True)
def draw_hierarchy(data_files: Dict[int, File]): neurons = common_axis([DataFrame.load(x['spike']) for x in files.values()]) for (day_id, data_file), neuron in zip(files.items(), neurons): lever = load_mat(data_file['response']) corr_mat = noise_autocorrelation(lever, neuron, data_file.attrs['frame_rate']) with Figure() as (ax,): ax.set_title(f"day-{day_id:02d}") fancy_dendrogram(linkage(corr_mat, 'average'), ax=ax)
def create_like(self: T, values: np.ndarray, axes: Optional[List[np.ndarray]] = None) -> T: new_obj = self.__class__( DataFrame(values, (axes if axes is not None else self.axes)), deepcopy(self.stimulus), self.sample_rate) new_obj.set_trials(self.trial_anchors, self.pre_time, self.post_time) new_obj.initialized = self.initialized return new_obj
def neuron_lever_unsampled(data_file: File, params: Dict[str, float]) -> Tuple[np.ndarray, np.ndarray]: """ Returns: neural_activity, lever """ lever = get_trials(data_file) neurons = fold_by(DataFrame.load(data_file["spike"]), lever, data_file.attrs['frame_rate'], True) neurons, lever = filter_empty_trial_sets(neurons.values, lever.values[0]) mask, filtered = devibrate_trials(lever, params["pre_time"]) return neurons[:, mask, :], lever[mask, :]
def draw_classify_neurons(data_file: File, neuron_ids: Optional[np.ndarray] = None): lever = load_mat(data_file['response']) neuron = DataFrame.load(data_file['spike']) if neuron_ids is not None: neuron = neuron[search_ar(neuron_ids, neuron.axes[0]), :] neuron_rate = data_file.attrs['frame_rate'] corr = motion_corr(lever, neuron, neuron_rate, 16000, motion_params) good, bad, anti = [corr[x, 0] for x in classify_cells(corr, 0.001)] with Figure(join(img_folder, "good_unrelated_cmp.svg"), (4, 6)) as ax: ax[0].bar((0, 1), [good.mean(), np.r_[bad, anti].mean()], yerr=[_sem(good), _sem(np.r_[bad, anti])])
def all_traces(record_file: File, ax: Axes): """plot full traces of all neurons and trial onsets""" lever_trajectory = load_mat(record_file["response"]) calcium_trace = _scale(DataFrame.load(record_file["measurement"]).values) time = np.linspace(0, lever_trajectory.shape[1] / lever_trajectory.sample_rate, lever_trajectory.shape[1]) ax.plot(time, _scale(lever_trajectory.values[0]) - 5, COLORS[1]) for idx, row in enumerate(calcium_trace): ax.plot(time, row + idx * 5) for point in lever_trajectory.timestamps / lever_trajectory.sample_rate: # trial onsets ax.axvline(x=point, color=COLORS[2])
def draw_rasterplot(day: int, data_file: File, neuron_id: int, params: MotionParams): lever = load_mat(data_file['response']) lever.center_on('motion', **params) neurons = DataFrame.load(data_file['spike']) neuron_rate = data_file.attrs['frame_rate'] traces = fold_by(neurons, lever, neuron_rate, True)[np.flatnonzero(neurons.axes[0] == neuron_id)[0], :, :] mask = np.all(traces.values > 0, axis=1) onset = int(round(params['pre_time'] * neuron_rate)) with Figure(join(img_folder, 'neuron-trace', f"raster-day-{day}.svg"), (2, 4)) as (ax,): labeled_heatmap(ax, traces[mask, :] - traces[mask, 0: onset].mean(axis=1, keepdims=True), cmap="coolwarm") ax.set_title(f"day-{day}")
def make_related_neurons( trial_log: SparseRec, spike_framerate: Tuple[Dict[str, np.ndarray], float]) -> np.ndarray: """Calcualte the pearson r between neuron activity and trajectory. Returns an array of p values for each neuron.""" spike, frame_rate = spike_framerate trial_spikes = fold_by(DataFrame.load(spike), trial_log, frame_rate, False) trajectory = scale(trial_log.values).ravel() p_values = [ pearsonr(neuron, trajectory)[1] for neuron in scale(trial_spikes.values).reshape( trial_spikes.shape[0], -1) ] return np.array(p_values)
def load_mat(file_name: Union[str, Dict[str, dict]]) -> SparseRec: if isinstance(file_name, str): data_dict = cell2dict(loadmat(file_name)['data']) else: data_dict = file_name['data'] samples_rate: float = data_dict['card']['ai_fs'] # type: ignore stimulus = _convert_psychsr_lever(data_dict) # type: ignore trace = _calculate_full_trace(data_dict).reshape(1, -1) # type: ignore axes = [ np.array(["right-lever"]), np.arange(trace.shape[1]) / samples_rate ] return SparseRec(DataFrame(trace, axes), stimulus, samples_rate)
def draw_neuron(day: int, data_file: File, neuron_id: int, params: MotionParams): """Draw one neuron in trial for one session, with bootstrapped spread as shadow.""" lever = load_mat(data_file['response']) lever.center_on("motion", **params) neuron = DataFrame.load(data_file['spike']) traces = fold_by(neuron, lever, data_file.attrs['frame_rate']) traces = traces[np.flatnonzero(traces.axes[0] == neuron_id)[0], :, :] mask = np.all(traces.values > 0, axis=1) pre_value = traces.values[mask, 0: int(round(params['pre_time'] * lever.sample_rate))].mean(axis=1, keepdims=True) trace_values = traces.values[mask, :] - pre_value with Figure(join(img_folder, "neuron-trace", f"day-{day:02d}.svg"), (1, 4)) as (ax,): tsplot(ax, trace_values, time=traces.axes[2], color=COLORS[4]) ax.set_title(f"day_{day:02d}")
def draw_neuron_corr(data_files: Dict[int, File], params: MotionParams, fov_id: str = None): neurons = common_axis([DataFrame.load(x['spike']) for x in data_files.values()]) last_day = max(data_files.keys()) lever = load_mat(data_files[last_day]['response']) neuron_rate = data_files[last_day].attrs['frame_rate'] good, bad, anti = classify_cells(motion_corr( lever, neurons[-1], neuron_rate, 16000, params), 0.001) result_list = list() for (day, data_file), neuron in zip(data_files.items(), neurons): lever.center_on('motion') # type: ignore motion_neurons = fold_by(neuron, lever, neuron_rate, True) result_list.append([reliability(motion_neuron) for motion_neuron in motion_neurons.values]) result = np.array(result_list) with Figure(join(img_folder, ("neuron_corr.svg" if fov_id is None else f"{fov_id}.svg"))) as ax: ax[0].plot(list(data_files.keys()), result[:, good])
def align_XY(spike_sample_rate: Tuple[DataFrame, int], filterd_log: SparseRec) -> Tuple[DataFrame, SparseRec]: """ Returns: X: spikes scaled y: lever trajectory resampled to the sample rate of spikes """ spike, sample_rate = spike_sample_rate resampled_trace = InterpolatedUnivariateSpline( filterd_log.axes[1], filterd_log.values[0])(spike['y']) y = filterd_log.create_like(scale_features(resampled_trace), [spike['y']]) # type: ignore y.sample_rate = sample_rate spike_df = DataFrame(scale_features(spike['data'], axes=1), [spike['x'], spike['y']]) # type: ignore return spike_df, y
def show_correspondance(ax: Axes, record_file: File, motion_params: Dict[str, float]): def scale(x): x -= x.mean() x /= x.std() return x lever = load_mat(record_file['response']) lever.center_on("motion", **motion_params) activity = DataFrame.load(record_file["measurement"]) neuron_rate = record_file.attrs['frame_rate'] trials = filter_empty_trials(ts.fold_by(activity, lever, neuron_rate, True)) slow_lever = ts.resample(lever.fold_trials().values, lever.sample_rate, neuron_rate, -1)[:, 0:trials.shape[1], :] ax.plot(scale(trials.values[0].reshape(-1)), 'green') ax.plot(scale(slow_lever.reshape(-1)), 'red')
def draw_network_graph(data_files: Dict[int, File], params: MotionParams, threshold: int = 16000): """Draw neuron functional connection for each session, with neurons colored by the last session. Args: data_files: {day_id: int, data_file: File} params: classify_cells need ["quiet_var", "window_size", "event_thres", "pre_time"] threshold: threshold for motion_corr, single linked cluster distance """ last_day = data_files[max(data_files.keys())] neurons = common_axis([DataFrame.load(x['spike']) for x in data_files.values()]) neuron_rate = last_day.attrs['frame_rate'] final_corr_mat = noise_autocorrelation(load_mat(last_day['response']), neurons[-1], neuron_rate) categories = classify_cells(motion_corr(last_day, neurons[-1], neuron_rate, threshold, params), 0.001) layout = corr_graph.get_layout(final_corr_mat, neurons[-1].axes[0]) for (day_id, data_file), neuron in zip(data_files.items(), neurons): corr_mat = noise_autocorrelation(load_mat(data_file['response']), neuron, neuron_rate) with Figure(join(img_folder, f"network-day-{day_id:02d}.svg")) as ax: corr_graph.corr_plot(ax[0], corr_mat, categories, neuron.axes[0], layout=layout) print('done')
def example_traces(ax: Axes, record_file: File, start: float, end: float, cells: Set[int]): """Visualize calcium trace of cells and the lever trajectory""" lever_trajectory = load_mat(record_file["response"]) calcium_trace = DataFrame.load(record_file["measurement"]) neuron_rate = record_file.attrs['frame_rate'] l_start, l_end = np.rint(np.multiply([start, end], lever_trajectory.sample_rate)).astype(np.int_) c_start, c_end = np.rint(np.multiply([start, end], neuron_rate)).astype(np.int_) ax.plot(np.linspace(0, l_end - l_start, l_end - l_start), # lever trajectory _scale(lever_trajectory.values[0][l_start: l_end]), COLORS[1]) time = np.linspace(0, calcium_trace.shape[1] / neuron_rate, lever_trajectory.shape[1]) spacing = iter(range(0, 500, 2)) for idx, row in enumerate(calcium_trace.values): if idx in cells: ax.plot(time[c_start: c_end] - l_start, _scale(row[c_start: c_end]) + next(spacing)) stim_onsets = lever_trajectory.timestamps[ (lever_trajectory.timestamps > l_start) & (lever_trajectory.timestamps < l_end)]\ / lever_trajectory.sample_rate - l_start for x in stim_onsets: ax.axvline(x=x, color=COLORS[2])
def run_amp_power(data_file: File) -> Tuple[float, float, float, float]: """Try to decode the max lever trajectory amplitude of each trial. Returns: pre_amp_power: mutual info between predicted (from neuron activity before motor onset) and real amplitude of trials in one session post_amp_power: mutual info between predicted (from neuron activity before motor onset) and real amplitude of trials in one session """ VALIDATE_FOLD = 10 lever = get_trials(data_file, motion_params) neuron = DataFrame.load(data_file['spike']) resampled_onsets = np.rint(lever.trial_anchors * (5 / 256)).astype(np.int_) - 3 folded = np.stack( [take_segment(trace, resampled_onsets, 6) for trace in neuron.values]) mask, filtered = devibrate_trials(lever.values, motion_params['pre_time']) mask &= np.any(folded > 0, axis=(0, 2)) amp = filtered[mask, 25:64].max(axis=1) - filtered[mask, 0:15].mean(axis=1) speed = np.diff(filtered[mask, 5:50], axis=1).max(axis=1) svr_rbf = SVR('rbf', 3, 1E-7, cache_size=1000) X = folded[:, mask, 0:3].swapaxes(0, 1).reshape(mask.sum(), -1) pre_amp_hat = cross_predict( X.T, amp, lambda x, y, y_t: svr_rbf.fit(x.T, y).predict(y_t.T), VALIDATE_FOLD, False) pre_v_hat = cross_predict( X.T, speed, lambda x, y, y_t: svr_rbf.fit(x.T, y).predict(y_t.T), VALIDATE_FOLD, False) X = folded[:, mask, 3:].swapaxes(0, 1).reshape(mask.sum(), -1) post_amp_hat = cross_predict( X.T, amp, lambda x, y, y_t: svr_rbf.fit(x.T, y).predict(y_t.T), VALIDATE_FOLD, False) post_v_hat = cross_predict( X.T, speed, lambda x, y, y_t: svr_rbf.fit(x.T, y).predict(y_t.T), VALIDATE_FOLD, False) return (mutual_info(pre_amp_hat, amp), mutual_info(post_amp_hat, amp), mutual_info(pre_v_hat, speed), mutual_info(post_v_hat, speed))
ax2.errorbar(day_ids, [np.mean(x) for x in amp], [_sem(x) for x in amp], color=COLORS[-1]) ax.set_title(str(neuron_id)) ax.legend() # Cell: Mesuare the inter-cell correlation between trials of typical pushes for single neurons on different days def draw_neuron_corr(data_files: Dict[int, File], params: MotionParams, fov_id: str = None): neurons = common_axis([DataFrame.load(x['spike']) for x in data_files.values()]) last_day = max(data_files.keys()) lever = load_mat(data_files[last_day]['response']) neuron_rate = data_files[last_day].attrs['frame_rate'] good, bad, anti = classify_cells(motion_corr( lever, neurons[-1], neuron_rate, 16000, params), 0.001) result_list = list() for (day, data_file), neuron in zip(data_files.items(), neurons): lever.center_on('motion') # type: ignore motion_neurons = fold_by(neuron, lever, neuron_rate, True) result_list.append([reliability(motion_neuron) for motion_neuron in motion_neurons.values]) result = np.array(result_list) with Figure(join(img_folder, ("neuron_corr.svg" if fov_id is None else f"{fov_id}.svg"))) as ax: ax[0].plot(list(data_files.keys()), result[:, good]) ## actual running common_id = common_axis([DataFrame.load(x['spike']) for x in files.values()])[-1].axes[0] draw_classify_neurons(files[14], common_id) draw_hierarchy(files) draw_stacked_bar(toml.load(join(res_folder, 'cluster.toml'))) # type: ignore neuron_ids = toml.load(join(res_folder, "0304-neurons.toml"))['neuron_id'] draw_noise(files, 27, motion_params) draw_neuron_corr(files, motion_params) ##
def make_trial_neuron( trial_log: SparseRec, spike_framerate: Tuple[Dict[str, np.ndarray], float]) -> DataFrame: spikes, frame_rate = spike_framerate # trial_neurons should be [neuron, trial, time_points] return fold_by(DataFrame.load(spikes), trial_log, frame_rate, True)