memmap_folder=memap_folder,
save_for_spikedetekt=False,
channels_to_save=None,
return_sliced_data=False)
pl.save(os.path.join(memap_folder, 'B14R9_raw.npy'), data)
data_ecog = data[:64, :]
data_probe = data[64:, :]
data_probe_hp = pl.memmap(os.path.join(memap_folder, 'data_probe_hp.dat'),
dtype='int16',
mode='w+',
shape=pl.shape(data_probe))
for i in pl.arange(0, pl.shape(data_probe)[0]):
data_probe_hp[i, :] = filters.high_pass_filter(data_probe[i, :],
Fsampling=f_sampling,
Fcutoff=f_hp_cutoff)
data_probe_hp.flush()
print(i)
pl.save(os.path.join(memap_folder, 'data_probe_hp.npy'), data_probe_hp)
shape_data_ss = (pl.shape(data_ecog)[0],
pl.shape(data_ecog)[1] / int(f_sampling / f_subsample))
data_ecog_lp_ss = pl.memmap(os.path.join(memap_folder, 'data_ecog_lp_ss.dat'),
dtype='int16',
mode='w+',
shape=shape_data_ss)
for i in pl.arange(0, pl.shape(data_ecog)[0]):
data_ecog_lp_ss[i, :] = signal.decimate(
filters.low_pass_filter(data_ecog[i, :],
Fsampling=f_sampling,
num_of_points_for_baseline = 10
selected_spikes_data_cube = tsne_cluster.create_data_cube_from_raw_extra_data(raw_data_ivm, data_cube_filename,
num_ivm_channels,
num_of_points_in_spike_trig,
cube_type, spike_times_of_indices_all,
num_of_points_in_spike_trig-1)
selected_spikes_concatenated = np.reshape(selected_spikes_data_cube, ((num_ivm_channels *
num_of_points_in_spike_trig),
len(spike_times_of_indices_all))).transpose()
selected_spikes_data_cube_filtered = np.zeros((num_ivm_channels, num_of_points_in_spike_trig,
len(spike_times_of_indices_all)))
for spike in np.arange(len(spike_times_of_indices_all)):
selected_spikes_data_cube_filtered[:, :, spike] = filters.high_pass_filter(selected_spikes_data_cube[:, :, spike],
30000, 30000 / 128) # removes the slant in each trial
selected_spikes_concatenated_filtered = np.reshape(selected_spikes_data_cube_filtered, ((num_ivm_channels *
num_of_points_in_spike_trig),
len(spike_times_of_indices_all))).transpose()
from sklearn.neural_network import BernoulliRBM as RBM
X = np.zeros((selected_spikes_concatenated_filtered.shape[0], selected_spikes_concatenated_filtered.shape[1]))
for i in np.arange(selected_spikes_concatenated_filtered.shape[0]):
X[i, :] = (selected_spikes_concatenated_filtered[i, :] - np.min(selected_spikes_concatenated_filtered[i, :], 0)) / (np.max(selected_spikes_concatenated_filtered[i, :], 0) - np.min(selected_spikes_concatenated_filtered[i, :], 0)) # 0-1 scaling
rbm = RBM(n_components=200, learning_rate=0.05, batch_size=100, n_iter=500)
rbm.fit(X.transpose())
rbm_features_tsne_selected_indices = TSNE.t_sne(rbm.components_.transpose(),
phases_all_shaftC = pl.load(os.path.join(memap_folder, 'phases_all_shaftC.npy'), mmap_mode=None)
data = pl.load(os.path.join(memap_folder,'B14R9_raw.npy'), mmap_mode='r+')
#----------Data generation-----------------
data = lio.read_all_csc(folder, assume_same_fs=False, memmap=True, memmap_folder=memap_folder, save_for_spikedetekt=False, channels_to_save=None, return_sliced_data=False)
pl.save(os.path.join(memap_folder, 'B14R9_raw.npy'), data)
data_ecog = data[:64,:]
data_probe = data[64:,:]
data_probe_hp = pl.memmap(os.path.join(memap_folder,'data_probe_hp.dat'), dtype='int16', mode='w+', shape=pl.shape(data_probe))
for i in pl.arange(0, pl.shape(data_probe)[0]):
data_probe_hp[i,:] = filters.high_pass_filter(data_probe[i,:], Fsampling=f_sampling, Fcutoff=f_hp_cutoff)
data_probe_hp.flush()
print(i)
pl.save(os.path.join(memap_folder, 'data_probe_hp.npy'), data_probe_hp)
shape_data_ss = (pl.shape(data_ecog)[0], pl.shape(data_ecog)[1]/int(f_sampling/f_subsample))
data_ecog_lp_ss = pl.memmap(os.path.join(memap_folder, 'data_ecog_lp_ss.dat'), dtype='int16', mode='w+', shape=shape_data_ss)
for i in pl.arange(0, pl.shape(data_ecog)[0]):
data_ecog_lp_ss[i,:] = signal.decimate(filters.low_pass_filter(data_ecog[i,:], Fsampling=f_sampling, Fcutoff=f_lp_cutoff), int(f_sampling/f_subsample))
data_ecog_lp_ss.flush()
print(i)
pl.save(os.path.join(memap_folder, 'data_ecog_lp_ss.npy'), data_ecog_lp_ss)
selected_spikes_data_cube = tsne_cluster.create_data_cube_from_raw_extra_data(
raw_data_ivm, data_cube_filename, num_ivm_channels,
num_of_points_in_spike_trig, cube_type, spike_times_of_indices_all,
num_of_points_in_spike_trig - 1)
selected_spikes_concatenated = np.reshape(
selected_spikes_data_cube,
((num_ivm_channels * num_of_points_in_spike_trig),
len(spike_times_of_indices_all))).transpose()
selected_spikes_data_cube_filtered = np.zeros(
(num_ivm_channels, num_of_points_in_spike_trig,
len(spike_times_of_indices_all)))
for spike in np.arange(len(spike_times_of_indices_all)):
selected_spikes_data_cube_filtered[:, :, spike] = filters.high_pass_filter(
selected_spikes_data_cube[:, :, spike], 30000,
30000 / 128) # removes the slant in each trial
selected_spikes_concatenated_filtered = np.reshape(
selected_spikes_data_cube_filtered,
((num_ivm_channels * num_of_points_in_spike_trig),
len(spike_times_of_indices_all))).transpose()
from sklearn.neural_network import BernoulliRBM as RBM
X = np.zeros((selected_spikes_concatenated_filtered.shape[0],
selected_spikes_concatenated_filtered.shape[1]))
for i in np.arange(selected_spikes_concatenated_filtered.shape[0]):
X[i, :] = (selected_spikes_concatenated_filtered[i, :] -
np.min(selected_spikes_concatenated_filtered[i, :], 0)) / (
np.max(selected_spikes_concatenated_filtered[i, :], 0) -
def time_lock_raw_data(data, events, times_to_cut, sampling_freq, baseline_time=None, sub_sample_freq=None,
high_pass_cutoff=None, rectify=False, low_pass_cutoff=None, avg_reref=False, keep_trials=False):
"""
Time locks, baselines, high or low passes, and sub samples the data (in that order)
"""
if np.ndim(events) == 2:
events = events[0, :]
number_of_trials = np.size(events, np.ndim(events) - 1)
times_to_cut = np.array(times_to_cut)
samples_to_cut = (times_to_cut * sampling_freq).astype(int)
if np.size(np.shape(data)) > 1:
number_of_channels = np.shape(data)[0]
else:
number_of_channels = 1
number_of_samples = samples_to_cut[1] - samples_to_cut[0]
time_axis = np.arange(times_to_cut[0], times_to_cut[1], (times_to_cut[1] - times_to_cut[0]) / number_of_samples)
if sub_sample_freq:
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data = np.zeros(
[number_of_channels, math.ceil(number_of_samples * (sub_sample_freq / sampling_freq)),
number_of_trials])
else:
tl_singletrial_data = np.zeros(
[math.ceil(number_of_samples * (sub_sample_freq / sampling_freq)), number_of_trials])
tl_avgtrials_data = np.empty(
[number_of_channels, math.ceil(number_of_samples * (sub_sample_freq / sampling_freq))])
else:
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data = np.zeros([number_of_channels, number_of_samples, number_of_trials])
else:
tl_singletrial_data = np.zeros([number_of_samples, number_of_trials])
tl_avgtrials_data = np.zeros([number_of_channels, number_of_samples])
for index in range(number_of_trials):
temp_samples_to_cut = samples_to_cut + events[index]
breakpoint = False
# if there aren't enough points at the begining of the data set then disregard the event and move to the next one
while np.min(temp_samples_to_cut) < 0:
if temp_samples_to_cut[0] < 0:
index = index + 1
temp_samples_to_cut = samples_to_cut + events[index]
elif temp_samples_to_cut[1] < 0:
breakpoint = True
if breakpoint:
break
if np.size(np.shape(data)) > 1:
temp_data = data[:, int(temp_samples_to_cut[0]): int(temp_samples_to_cut[1])]
else:
temp_data = data[int(temp_samples_to_cut[0]): int(temp_samples_to_cut[1])]
if avg_reref: # rereference with mean over all channels
temp_data = temp_data - np.mean(temp_data, 0)
if high_pass_cutoff:
temp_data = filt.high_pass_filter(temp_data, sampling_freq, high_pass_cutoff)
elif low_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq, low_pass_cutoff)
if rectify:
temp_data = np.abs(temp_data)
if low_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq, low_pass_cutoff)
else:
temp_data = filt.low_pass_filter(temp_data, sampling_freq, high_pass_cutoff / 2)
elif not rectify and high_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq, high_pass_cutoff / 2)
if sub_sample_freq:
temp_data, sub_time_axis = subsample_data(temp_data, time_axis, sampling_freq, sub_sample_freq,
filterType='fir', filterOrder=30)
if baseline_time is not None:
if sub_sample_freq:
temp_data = baseline_correct(temp_data, sub_sample_freq, time_axis, baseline_time[0], baseline_time[1])
else:
temp_data = baseline_correct(temp_data, sampling_freq, time_axis, baseline_time[0], baseline_time[1])
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data[:, :, index] = temp_data
else:
tl_singletrial_data[:, index] = temp_data
if index == 0:
tl_avgtrials_data = temp_data
elif index > 0:
tl_avgtrials_data += temp_data
if index % 10 == 0:
print(index)
tl_avgtrials_data /= number_of_trials
if sub_sample_freq:
time_axis = sub_time_axis
returned_tuple = [tl_avgtrials_data, time_axis]
if keep_trials:
returned_tuple = [tl_singletrial_data, tl_avgtrials_data, time_axis]
return returned_tuple
def time_lock_raw_data(data,
events,
times_to_cut,
sampling_freq,
baseline_time=None,
sub_sample_freq=None,
high_pass_cutoff=None,
rectify=False,
low_pass_cutoff=None,
avg_reref=False,
keep_trials=False):
"""
Time locks, baselines, high or low passes, and sub samples the data (in that order)
"""
if np.ndim(events) == 2:
events = events[0, :]
number_of_trials = np.size(events, np.ndim(events) - 1)
times_to_cut = np.array(times_to_cut)
samples_to_cut = (times_to_cut * sampling_freq).astype(int)
if np.size(np.shape(data)) > 1:
number_of_channels = np.shape(data)[0]
else:
number_of_channels = 1
number_of_samples = samples_to_cut[1] - samples_to_cut[0]
time_axis = np.arange(times_to_cut[0], times_to_cut[1],
(times_to_cut[1] - times_to_cut[0]) /
number_of_samples)
if sub_sample_freq:
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data = np.zeros([
number_of_channels,
math.ceil(number_of_samples *
(sub_sample_freq / sampling_freq)),
number_of_trials
])
else:
tl_singletrial_data = np.zeros([
math.ceil(number_of_samples *
(sub_sample_freq / sampling_freq)),
number_of_trials
])
tl_avgtrials_data = np.empty([
number_of_channels,
math.ceil(number_of_samples * (sub_sample_freq / sampling_freq))
])
else:
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data = np.zeros(
[number_of_channels, number_of_samples, number_of_trials])
else:
tl_singletrial_data = np.zeros(
[number_of_samples, number_of_trials])
tl_avgtrials_data = np.zeros([number_of_channels, number_of_samples])
for index in range(number_of_trials):
temp_samples_to_cut = samples_to_cut + events[index]
breakpoint = False
# if there aren't enough points at the begining of the data set then disregard the event and move to the next one
while np.min(temp_samples_to_cut) < 0:
if temp_samples_to_cut[0] < 0:
index = index + 1
temp_samples_to_cut = samples_to_cut + events[index]
elif temp_samples_to_cut[1] < 0:
breakpoint = True
if breakpoint:
break
if np.size(np.shape(data)) > 1:
temp_data = data[:,
int(temp_samples_to_cut[0]
):int(temp_samples_to_cut[1])]
else:
temp_data = data[int(temp_samples_to_cut[0]
):int(temp_samples_to_cut[1])]
if avg_reref: # rereference with mean over all channels
temp_data = temp_data - np.mean(temp_data, 0)
if high_pass_cutoff:
temp_data = filt.high_pass_filter(temp_data, sampling_freq,
high_pass_cutoff)
elif low_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq,
low_pass_cutoff)
if rectify:
temp_data = np.abs(temp_data)
if low_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq,
low_pass_cutoff)
else:
temp_data = filt.low_pass_filter(temp_data, sampling_freq,
high_pass_cutoff / 2)
elif not rectify and high_pass_cutoff:
temp_data = filt.low_pass_filter(temp_data, sampling_freq,
high_pass_cutoff / 2)
if sub_sample_freq:
temp_data, sub_time_axis = subsample_data(temp_data,
time_axis,
sampling_freq,
sub_sample_freq,
filterType='fir',
filterOrder=30)
if baseline_time is not None:
if sub_sample_freq:
temp_data = baseline_correct(temp_data, sub_sample_freq,
time_axis, baseline_time[0],
baseline_time[1])
else:
temp_data = baseline_correct(temp_data, sampling_freq,
time_axis, baseline_time[0],
baseline_time[1])
if keep_trials:
if np.size(np.shape(data)) > 1:
tl_singletrial_data[:, :, index] = temp_data
else:
tl_singletrial_data[:, index] = temp_data
if index == 0:
tl_avgtrials_data = temp_data
elif index > 0:
tl_avgtrials_data += temp_data
if index % 10 == 0:
print(index)
tl_avgtrials_data /= number_of_trials
if sub_sample_freq:
time_axis = sub_time_axis
returned_tuple = [tl_avgtrials_data, time_axis]
if keep_trials:
returned_tuple = [tl_singletrial_data, tl_avgtrials_data, time_axis]
return returned_tuple
