def select_spikes_in_certain_channels(spike_threshold, raw_data_file, common_spikes, indices_of_common_spikes_in_extra,
good_channels, num_of_raw_data_channels):
"""
Select spikes that appear only on the good_channels
Parameters
----------
spike_threshold
raw_data_file
common_spikes
indices_of_common_spikes_in_extra
good_channels
num_of_raw_data_channels
Returns
-------
"""
raw_data = ephys.load_raw_data(filename=raw_data_file, numchannels=num_of_raw_data_channels)
common_spikes = np.array(common_spikes)
indices_of_common_spikes_in_extra = np.array(indices_of_common_spikes_in_extra)
t = raw_data.dataMatrix[:, common_spikes]
if spike_threshold > 0:
spike_channels = np.argmin(t, axis=0)
if spike_threshold < 0:
spike_channels = np.argmax(t, axis=0)
good_spike_indices = [i for i, x in list(enumerate(spike_channels)) if np.in1d(x, good_channels)]
common_spikes = common_spikes[good_spike_indices]
indices_of_common_spikes_in_extra = indices_of_common_spikes_in_extra[good_spike_indices]
return common_spikes, indices_of_common_spikes_in_extra
def select_spikes_in_certain_channels(spike_thresholds, common_spikes, indices_of_common_spikes_in_extra,
good_channels, num_of_raw_data_channels):
raw_data = ioep.load_raw_data(filename=os.path.join(data_folder, 'amplifier'+date+'T'+cell_capture_times+'.bin'),
numchannels=num_of_raw_data_channels)
common_spikes = np.array(common_spikes)
indices_of_common_spikes_in_extra = np.array(indices_of_common_spikes_in_extra)
t = raw_data.dataMatrix[:, common_spikes]
if spike_thresholds > 0:
spike_channels = np.argmin(t, axis=0)
if spike_thresholds < 0:
spike_channels = np.argmax(t, axis=0)
good_spike_indices = [i for i,x in list(enumerate(spike_channels)) if np.in1d(x, good_channels)]
common_spikes = common_spikes[good_spike_indices]
indices_of_common_spikes_in_extra = indices_of_common_spikes_in_extra[good_spike_indices]
return common_spikes, indices_of_common_spikes_in_extra
def select_spikes_in_certain_channels(spike_thresholds, common_spikes, indices_of_common_spikes_in_extra,
good_channels, num_of_raw_data_channels):
raw_data = ioep.load_raw_data(filename=os.path.join(data_folder, 'amplifier'+date+'T'+cell_capture_times+'.bin'),
numchannels=num_of_raw_data_channels)
common_spikes = np.array(common_spikes)
indices_of_common_spikes_in_extra = np.array(indices_of_common_spikes_in_extra)
t = raw_data.dataMatrix[:, common_spikes]
if spike_thresholds > 0:
spike_channels = np.argmin(t, axis=0)
if spike_thresholds < 0:
spike_channels = np.argmax(t, axis=0)
good_spike_indices = [i for i,x in list(enumerate(spike_channels)) if np.in1d(x, good_channels)]
common_spikes = common_spikes[good_spike_indices]
indices_of_common_spikes_in_extra = indices_of_common_spikes_in_extra[good_spike_indices]
return common_spikes, indices_of_common_spikes_in_extra
def select_spikes_in_certain_channels(spike_threshold, raw_data_file,
common_spikes,
indices_of_common_spikes_in_extra,
good_channels, num_of_raw_data_channels):
"""
Select spikes that appear only on the good_channels
Parameters
----------
spike_threshold
raw_data_file
common_spikes
indices_of_common_spikes_in_extra
good_channels
num_of_raw_data_channels
Returns
-------
"""
raw_data = ephys.load_raw_data(filename=raw_data_file,
numchannels=num_of_raw_data_channels)
common_spikes = np.array(common_spikes)
indices_of_common_spikes_in_extra = np.array(
indices_of_common_spikes_in_extra)
t = raw_data.dataMatrix[:, common_spikes]
if spike_threshold > 0:
spike_channels = np.argmin(t, axis=0)
if spike_threshold < 0:
spike_channels = np.argmax(t, axis=0)
good_spike_indices = [
i for i, x in list(enumerate(spike_channels))
if np.in1d(x, good_channels)
]
common_spikes = common_spikes[good_spike_indices]
indices_of_common_spikes_in_extra = indices_of_common_spikes_in_extra[
good_spike_indices]
return common_spikes, indices_of_common_spikes_in_extra
cell_capture_times = all_cell_capture_times[rat]
adc_channel_used = 0
adc_dtype = np.uint16
inter_spike_time_distance = 0.01
amp_gain = 100
num_ivm_channels = 32
amp_dtype = np.uint16
# 1) Generate the .dat file
filename_raw_data = join(data_folder,
'amplifier{}T{}.bin'.format(dates[rat], all_cell_capture_times[rat][cell]))
raw_data = ioep.load_raw_data(filename=filename_raw_data, numchannels=num_ivm_channels, dtype=amp_dtype)
filename_kl_data = join(analysis_folder, r'klustakwik_cell{}\raw_data_klusta.dat'.format(cell))
iokl.make_dat_file(raw_data=raw_data.dataMatrix, num_channels=num_ivm_channels, filename=filename_kl_data)
# Run t-sne
kwx_file_path = join(analysis_folder, 'klustakwik_cell{}'.format(cell),
r'threshold_6_5std/threshold_6_5std.kwx')
perplexity = 100
theta = 0.2
iterations = 2000
gpu_mem = 0.8
eta = 200
early_exaggeration = 4.0
indices_of_spikes_to_tsne = None#range(spikes_to_do)
sampling_freq = 30000
num_of_points_in_spike_trig = 64
num_of_points_for_baseline = 10
data_cube_filename = os.path.join(kilosort_path, 'tsne\\raw_data_cube_for_pcs.npy')
autocor_bin_number = 100
prb_file = r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\klustakwik\128ch_passive_imec.prb'
spike_indices_to_use = None
num_of_spikes = np.shape(template_features_tsne)[1]
shape_of_cut_extracellular_data = (num_ivm_channels, num_of_points_in_spike_trig, num_of_spikes)
raw_data_filename = os.path.join(base_dir, data_dir, 'amplifier2015-09-03T21_18_47.bin')
raw_data_ivm = ephys.load_raw_data(raw_data_filename, numchannels=num_ivm_channels, dtype=amp_dtype).dataMatrix
# manual clustering the kilosort data
spike_indices_to_use = None
num_of_spikes = len(spikes_used)
shape_of_cut_extracellular_data = (num_ivm_channels, num_of_points_in_spike_trig, num_of_spikes)
raw_data_filename = os.path.join(base_dir, data_dir, 'amplifier2015-09-03T21_18_47.bin')
raw_data_ivm = ephys.load_raw_data(raw_data_filename, numchannels=num_ivm_channels, dtype=amp_dtype).dataMatrix
tsne_cluster.gui_manual_cluster_tsne_spikes(tsne_array_or_filename=template_features_tsne,
spike_times_list_or_filename=np.reshape(spike_times_kilosort,
(len(spike_times_kilosort))),
raw_extracellular_data=None,
num_of_points_for_baseline=num_of_points_for_baseline,
#CCU data brain 2015-08-28
analysis_folder = r'Z:\j\Joana Neto\Backup_2017_28_06\PCDisk\Paper Impedance\128chNeuroseeker\CCU\2015_08_28\pair2.2'
#raw_data_file_ivm = os.path.join(analysis_folder + '\\' + 'amplifier2015-08-28T20_15_45.bin')
#raw_data_file_ivm = r"Z:\j\Joana Neto\Backup_2017_28_06\PCDisk\Data_128ch\2015-08-28\Data\amplifier2015-08-28T23_28_20.bin"
raw_data_file_ivm=r"S:\swc\kampff\j\Joana Neto\Backup_2017_28_06\PCDisk\dataset online\Validating 128ch\2015_08_28_Pair_2_0\2015_08_28_Pair_2.2\amplifier2015-08-28T20_15_45.bin"
#Open data
amp_dtype = np.uint16
Probe_y_digitization = 32768
num_ivm_channels = 128
sampling_freq = 30000
filtered_data_type = np.float64
voltage_step_size = 0.195e-6
scale_uV = 1000000
raw_data_ivm = ephys.load_raw_data(raw_data_file_ivm, numchannels=num_ivm_channels, dtype=amp_dtype)
temp_unfiltered = raw_data_ivm.dataMatrix
temp_unfiltered = temp_unfiltered.astype(filtered_data_type)
temp_unfiltered_uV = (temp_unfiltered - Probe_y_digitization) * scale_uV * voltage_step_size
#High-pass filter
high_pass_freq = 500
def highpass(data,BUTTER_ORDER=3, F_HIGH=14250,sampleFreq=30000.0,passFreq=100.0):
b, a = signal.butter(BUTTER_ORDER,(passFreq/(sampleFreq/2), F_HIGH/(sampleFreq/2)),'pass')
return signal.filtfilt(b,a,data)
#tfinal = np.ceil((temp_unfiltered_uV.shape[1])/2)
tfinal= 100000
#temp_filtered_uV = highpass(temp_unfiltered_uV[:,0:tfinal], F_HIGH = (sampling_freq/2)*0.95, sampleFreq = sampling_freq, passFreq = high_pass_freq)
temp_filtered_uV = highpass(temp_unfiltered_uV[:,0:tfinal], F_HIGH = (sampling_freq/2)*0.95, sampleFreq = sampling_freq, passFreq = high_pass_freq)
#Noise calculation
cell_capture_times = all_cell_capture_times[rat]
adc_channel_used = 0
adc_dtype = np.uint16
inter_spike_time_distance = 0.01
amp_gain = 100
num_ivm_channels = 32
amp_dtype = np.uint16
# 1) Generate the .dat file
filename_raw_data = join(
data_folder,
'amplifier{}T{}.bin'.format(dates[rat], all_cell_capture_times[rat][cell]))
raw_data = ioep.load_raw_data(filename=filename_raw_data,
numchannels=num_ivm_channels,
dtype=amp_dtype)
filename_kl_data = join(analysis_folder,
r'klustakwik_cell{}\raw_data_klusta.dat'.format(cell))
iokl.make_dat_file(raw_data=raw_data.dataMatrix,
num_channels=num_ivm_channels,
filename=filename_kl_data)
# Run t-sne
kwx_file_path = join(analysis_folder, 'klustakwik_cell{}'.format(cell),
r'threshold_6_5std/threshold_6_5std.kwx')
perplexity = 100
theta = 0.2
iterations = 2000
gpu_mem = 0.8
eta = 200
time_paw_event = 'time of event'
name_traj_point = 'name of trajectory point'
trial_traj_point = 'trial of trajectory point'
frame_traj_point = 'frame of trajectory point'
time_traj_point = 'time of trajectory point'
x_traj_point = 'X of trajectory point'
y_traj_point = 'Y of trajectory point'
session = pd.HDFStore(os.path.join(path, session_path))
paws = session[paw_events_key]
fl_paw = paws[[trial_paw_event, time_paw_event, flpaw]][paws[flpaw] != -1]
fl_paw_with_traj = fl_paw[[trial_paw_event, time_paw_event, flpaw]][paws[trial_paw_event] % 2 == 0]
sync = np.squeeze(ephys.load_raw_data(os.path.join(path, sync_path), numchannels=1, dtype=np.uint8).dataMatrix)
sync_diff = np.diff(sync.astype('int8'))
cam_shutter_closing_samples = np.squeeze((sync_diff < -0.9).nonzero())
start_sample = cam_shutter_closing_samples[fl_paw[flpaw].iloc[0]]
session.close()
# single paw cycle events
lfpaw_start_cycle_touch = pd.DataFrame([[fl_paw[trial_paw_event].iloc[0], fl_paw[flpaw].iloc[0], start_sample]], columns=['trial', 'frame', 'sample'])
for i in np.arange(0, fl_paw.shape[0] - 1):
trial = fl_paw[trial_paw_event].iloc[i]
next_trial = fl_paw[trial_paw_event].iloc[i+1]
if trial == next_trial:
next_frame = fl_paw[flpaw].iloc[i+1]
next_sample = cam_shutter_closing_samples[next_frame]
lfpaw_start_cycle_touch.loc[i+1] = [next_trial, next_frame, next_sample]
print(len(spike_triggers))
#Filter for extracellular recording
def highpass(data,BUTTER_ORDER=3, F_HIGH=14250,sampleFreq=30000.0,passFreq=100.0):
b, a = signal.butter(BUTTER_ORDER,(passFreq/(sampleFreq/2), F_HIGH/(sampleFreq/2)),'pass')
return signal.filtfilt(b,a,data)
# Generate the (channels x time_points x spikes) high passed extracellular recordings datasets for all cells
all_cells_ivm_filtered_data = {}
data_to_load = 't'
for i in np.arange(0, len(good_cells)):
raw_data_file_ivm = os.path.join(data_folder, 'amplifier'+date+'T'+cell_capture_times[good_cells[i]]+'.bin')
raw_data_ivm = ephys.load_raw_data(raw_data_file_ivm, numchannels=num_ivm_channels, dtype=amp_dtype)
num_of_spikes = len(all_cells_spike_triggers[good_cells[i]])
shape_of_filt_spike_trig_ivm = ((num_ivm_channels,
num_of_points_in_spike_trig_ivm,
num_of_spikes))
ivm_data_filtered = np.memmap(os.path.join(analysis_folder, types_of_data_to_load[data_to_load].format(good_cells[i])),
dtype=filtered_data_type,
mode='w+',
shape=shape_of_filt_spike_trig_ivm)
for spike in np.arange(0, num_of_spikes):
trigger_point = all_cells_spike_triggers[good_cells[i]][spike]
start_point = int(trigger_point - (num_of_points_in_spike_trig_ivm + num_of_points_for_padding)/2)
if start_point < 0:
break
data_cube_filename = os.path.join(kilosort_path,
'tsne\\raw_data_cube_for_pcs.npy')
autocor_bin_number = 100
prb_file = r'D:\Data\George\Projects\SpikeSorting\Joana_Paired_128ch\2015-09-03\Analysis\klustakwik\128ch_passive_imec.prb'
spike_indices_to_use = None
num_of_spikes = np.shape(template_features_tsne)[1]
shape_of_cut_extracellular_data = (num_ivm_channels,
num_of_points_in_spike_trig, num_of_spikes)
raw_data_filename = os.path.join(base_dir, data_dir,
'amplifier2015-09-03T21_18_47.bin')
raw_data_ivm = ephys.load_raw_data(raw_data_filename,
numchannels=num_ivm_channels,
dtype=amp_dtype).dataMatrix
# manual clustering the kilosort data
spike_indices_to_use = None
num_of_spikes = len(spikes_used)
shape_of_cut_extracellular_data = (num_ivm_channels,
num_of_points_in_spike_trig, num_of_spikes)
raw_data_filename = os.path.join(base_dir, data_dir,
'amplifier2015-09-03T21_18_47.bin')
raw_data_ivm = ephys.load_raw_data(raw_data_filename,
numchannels=num_ivm_channels,
dtype=amp_dtype).dataMatrix
inter_spike_time_distance = 0.002
amp_gain = 100
amp_dtype = np.uint16
good_channels = [8, 18, 13, 23, 28, 3, 9, 29, 2, 22]
# Generate the data phy requires
# 1) Generate the prb file
filename_prb = join(base_directory,
'Analysis\klustakwik_cell3\passive_cerebro_dense.prb')
all_electrodes, channel_positions = prnn.create_32channels_nn_prb(filename_prb)
# 2) Generate the .dat file
filename_raw_data = join(base_directory,
r'Data\amplifier2014-11-25T23_00_08.bin')
raw_data = ioep.load_raw_data(filename=filename_raw_data)
filename_kl_data = join(base_directory,
r'Analysis\klustakwik_cell3\raw_data_klusta.dat')
iokl.make_dat_file(raw_data=raw_data.dataMatrix,
num_channels=num_ivm_channels,
filename=filename_kl_data)
# Get extra spike times (as found by phy detect)
kwik_file_path = join(base_directory,
r'Analysis\klustakwik_cell3\threshold_6_5std',
'amplifier_depth_70um_s16.kwik')
h5file = h5.File(kwik_file_path, mode='r')
extra_spike_times = np.array(
list(h5file['channel_groups/0/spikes/time_samples']))
h5file.close()
time_paw_event = 'time of event'
name_traj_point = 'name of trajectory point'
trial_traj_point = 'trial of trajectory point'
frame_traj_point = 'frame of trajectory point'
time_traj_point = 'time of trajectory point'
x_traj_point = 'X of trajectory point'
y_traj_point = 'Y of trajectory point'
session = pd.HDFStore(os.path.join(path, session_path))
paws = session[paw_events_key]
fl_paw = paws[[trial_paw_event, time_paw_event, flpaw]][paws[flpaw] != -1]
fl_paw_with_traj = fl_paw[[trial_paw_event, time_paw_event, flpaw]][paws[trial_paw_event] % 2 == 0]
sync = np.squeeze(ephys.load_raw_data(os.path.join(path, sync_path), numchannels=1, dtype=np.uint8).dataMatrix)
sync_diff = np.diff(sync.astype('int8'))
cam_shutter_closing_samples = np.squeeze((sync_diff < -0.9).nonzero())
start_sample = cam_shutter_closing_samples[fl_paw[flpaw].iloc[0]]
session.close()
# single paw cycle events
lfpaw_start_cycle_touch = pd.DataFrame([[fl_paw[trial_paw_event].iloc[0], fl_paw[flpaw].iloc[0], start_sample]], columns=['trial', 'frame', 'sample'])
for i in np.arange(0, fl_paw.shape[0] - 1):
trial = fl_paw[trial_paw_event].iloc[i]
next_trial = fl_paw[trial_paw_event].iloc[i+1]
if trial == next_trial:
next_frame = fl_paw[flpaw].iloc[i+1]
next_sample = cam_shutter_closing_samples[next_frame]
lfpaw_start_cycle_touch.loc[i+1] = [next_trial, next_frame, next_sample]
import matplotlib.pyplot as plt
from tsne_for_spikesort_old import io_with_cpp as io
# DEFINE THE CELL TO WORK ON _________
recording = 'd5331'
cell = '01'
# ____________________________________
full_cell_name = recording + cell
base_folder = join('Z:\g\George\DataAndResults\Harris_JuxtaTetrodePairedData',
recording)
analysis_folder = join(base_folder, 'Analysis')
data = ephys.load_raw_data(join(base_folder, 'Data', full_cell_name + '.dat'),
numchannels=8,
dtype=np.int16)
ic_channel = 5
ec_channels = [1, 2, 3, 4]
# Make the binary data file for klustakwik--
extra_data = data.dataMatrix[ec_channels, :]
extra_data_to_bin = np.reshape(extra_data.T, (extra_data.size))
extra_data_to_bin.tofile(
join(analysis_folder, 'KlustaKwik', cell, full_cell_name + '_extra.dat'))
# ------------------------------------------
sampling_freq = 10000
juxta_spike_triggers, juxta_spike_peaks, juxta_spike_data_in_V = \