def make_unit_plots(file_dir, fs):
'''Makes waveform plots for sorted unit in unit_waveforms_plots
Parameters
----------
file_dir : str, full path to recording directory
fs : float, smapling rate in Hz
'''
h5_name = h5io.get_h5_filename(file_dir)
h5_file = os.path.join(file_dir, h5_name)
plot_dir = os.path.join(file_dir, 'unit_waveforms_plots')
if os.path.exists(plot_dir):
shutil.rmtree(plot_dir, ignore_errors=True)
os.mkdir(plot_dir)
fs_str = '(%g samples per ms)' % (fs / 1000.0)
unit_numbers = get_unit_numbers(h5_file)
with tables.open_file(h5_file, 'r+') as hf5:
units = hf5.list_nodes('/sorted_units')
for i, unit in zip(unit_numbers, units):
# plot all waveforms
waveforms = unit.waveforms[:]
descriptor = hf5.root.unit_descriptor[i]
fig, ax = blech_waveforms_datashader.waveforms_datashader(
waveforms)
ax.set_xlabel('Samples (%s)' % fs_str)
ax.set_ylabel('Voltage (microvolts)')
unit_title = (
('Unit %i, total waveforms = %i\nElectrode: %i, '
'Single Unit: %i, RSU: %i, FSU: %i') %
(i, waveforms.shape[0], descriptor['electrode_number'],
descriptor['single_unit'], descriptor['regular_spiking'],
descriptor['fast_spiking']))
ax.set_title(unit_title)
fig.savefig(os.path.join(plot_dir, 'Unit%i.png' % i))
plt.close('all')
# Plot mean and SEM of waveforms
# Downsample by 10 to remove upsampling from de-jittering
fig = plt.figure()
mean_wave = np.mean(waveforms[:, ::10], axis=0)
std_wave = np.std(waveforms[:, ::10], axis=0)
mean_x = np.arange(mean_wave.shape[0]) + 1
plt.plot(mean_x, mean_wave, linewidth=4.0)
plt.fill_between(mean_x,
mean_wave - std_wave,
mean_wave + std_wave,
alpha=0.4)
plt.xlabel('Samples (%s)' % fs_str)
plt.ylabel('Voltage (microvolts)')
plt.title(unit_title)
fig.savefig(os.path.join(plot_dir, 'Unit%i_mean_sd.png' % i))
plt.close('all')
def make_unit_plots(file_dir, unit_name, save_dir=None):
'''Makes waveform plots for sorted unit in unit_waveforms_plots
Parameters
----------
file_dir : str, full path to recording directory
fs : float, smapling rate in Hz
'''
if isinstance(unit_name, int):
unit_num = unit_name
unit_name = 'unit%03i' % unit_num
else:
unit_num = dio.h5io.parse_unit_number(unit_name)
waveforms, descriptor, fs = dio.h5io.get_unit_waveforms(
file_dir, unit_name)
fs_str = '%g samples per ms' % (fs / 10 / 1000.0
) # since both theses plots
# downsample by 10 and then to convert to samples/ms
fig, ax = blech_waveforms_datashader.waveforms_datashader(waveforms)
ax.set_xlabel('Samples (%s)' % fs_str)
ax.set_ylabel('Voltage (microvolts)')
unit_title = (('Unit %i, total waveforms = %i\nElectrode: %i, '
'Single Unit: %i, RSU: %i, FSU: %i') %
(unit_num, waveforms.shape[0],
descriptor['electrode_number'], descriptor['single_unit'],
descriptor['regular_spiking'], descriptor['fast_spiking']))
ax.set_title(unit_title)
fig.savefig(os.path.join(save_dir, 'Unit%i.png' % unit_num))
plt.close('all')
# Plot mean and SEM of waveforms
# Downsample by 10 to remove upsampling from de-jittering
fig, ax = plt.subplots(figsize=(12, 8))
mean_wave = np.mean(waveforms[:, ::10], axis=0)
std_wave = np.std(waveforms[:, ::10], axis=0)
mean_x = np.arange(mean_wave.shape[0]) + 1
ax.plot(mean_x, mean_wave, linewidth=4.0)
ax.fill_between(mean_x,
mean_wave - std_wave,
mean_wave + std_wave,
alpha=0.4)
ax.set_xlabel('Samples (%s)' % fs_str)
ax.set_ylabel('Voltage (microvolts)')
ax.set_title(unit_title)
fig.savefig(os.path.join(save_dir, 'Unit%i_mean_sd.png' % unit_num))
plt.close('all')
def plot_waveforms(waveforms, title=None, save_file=None, threshold=None):
'''Plots a cluster with isi and violation info for viewing
Parameters
----------
cluster : dict with cluster info
'''
fig, ax = blech_waveforms_datashader.waveforms_datashader(
waveforms, threshold=threshold)
ax.set_xlabel('Samples', fontsize=12)
ax.set_ylabel('Voltage (microvolts)', fontsize=12)
ax.set_title(title, fontsize=12)
plt.xticks(fontsize=10)
plt.yticks(fontsize=10)
if save_file is not None:
fig.savefig(save_file)
plt.close(fig)
return None, None
else:
return fig, ax
def plot_cluster(cluster, index=None):
'''Plots a cluster with isi and violation info for viewing
Parameters
----------
cluster : dict with cluster info
'''
fig, ax = blech_waveforms_datashader.waveforms_datashader(
cluster['spike_waveforms'])
ax.set_xlabel('Samples')
ax.set_ylabel('Voltage (microvolts)')
title_str = (('Cluster Name: %s\n2ms Violations=%0.1f%%, '
'1ms Violations=%0.1f%%\nNumber of Waveforms'
'=%i') %
(cluster['Cluster Name'], cluster['2ms_violations'],
cluster['1ms_violations'], cluster['spike_times'].shape[0]))
if index is not None:
title_str = 'Index: %i %s, ' % (index, title_str)
ax.set_title(title_str)
return fig, ax
def blech_clust_process(electrode_num, file_dir, params):
print('Clustering electrode %i...' % electrode_num)
# Check if the directories for this electrode number exist - if they do, delete
# them (existence of the directories indicates a job restart on the cluster, so
# restart afresh), and make the directories
fig_folders = [os.path.join('Plots',str(electrode_num)),
os.path.join('Plots',str(electrode_num),'Plots'),
os.path.join('spike_waveforms','electrode%i' % electrode_num),
os.path.join('spike_times','electrode%i' % electrode_num),
os.path.join('clustering_results','electrode%i' % electrode_num)]
for x in fig_folders:
tmp_path = os.path.join(file_dir, x)
if os.path.isdir(tmp_path):
shutil.rmtree(tmp_path)
os.mkdir(tmp_path)
# Get the names of all files in the current directory, and find the .params and hdf5 (.h5) file
file_list = os.listdir(file_dir)
h5_file = h5io.get_h5_filename(file_dir)
# Assign the parameters to variables
max_clusters = params['clustering_params']['Max Number of Clusters']
num_iter = params['clustering_params']['Max Number of Iterations']
thresh = params['clustering_params']['Convergence Criterion']
num_restarts = params['clustering_params']['GMM random restarts']
voltage_cutoff = params['data_params']['V_cutoff for disconnected headstage']
max_breach_rate = params['data_params']['Max rate of cutoff breach per second']
max_secs_above_cutoff = params['data_params']['Max allowed seconds with a breach']
max_mean_breach_rate_persec = params['data_params']['Max allowed breaches per second']
wf_amplitude_sd_cutoff = params['data_params']['Intra-cluster waveform amp SD cutoff']
bandpass_lower_cutoff = params['bandpass_params']['Lower freq cutoff']
bandpass_upper_cutoff = params['bandpass_params']['Upper freq cutoff']
spike_snapshot_before = params['spike_snapshot']['Time before spike (ms)']
spike_snapshot_after = params['spike_snapshot']['Time after spike (ms)']
sampling_rate = params['sampling_rate']
data_quality = params['data_quality']
# Open up hdf5 file, and load this electrode number
# Check if referenced data exists, if not grab raw
raw_el = h5io.get_referenced_trace(file_dir, electrode_num)
if raw_el is None:
raw_el = h5io.get_raw_trace(file_dir, electrode_num)
if raw_el is None:
raise KeyError('Neither /raw/electrode{0} nor /referenced/electrode{0} found in {1}'. \
format(electrode_num,hdf5_file))
# High bandpass filter the raw electrode recordings
filt_el = clust.get_filtered_electrode(raw_el, freq = [bandpass_lower_cutoff, bandpass_upper_cutoff], sampling_rate = sampling_rate)
# Delete raw electrode recording from memory
del raw_el
# Calculate the 3 voltage parameters
breach_rate = float(len(np.where(filt_el>voltage_cutoff)[0])*int(sampling_rate))/len(filt_el)
test_el = np.reshape(filt_el[:int(sampling_rate)*int(len(filt_el)/sampling_rate)], (-1, int(sampling_rate)))
breaches_per_sec = [len(np.where(test_el[i] > voltage_cutoff)[0]) for i in range(len(test_el))]
breaches_per_sec = np.array(breaches_per_sec)
secs_above_cutoff = len(np.where(breaches_per_sec > 0)[0])
if secs_above_cutoff == 0:
mean_breach_rate_persec = 0
else:
mean_breach_rate_persec = np.mean(breaches_per_sec[np.where(breaches_per_sec > 0)[0]])
# And if they all exceed the cutoffs, assume that the headstage fell off mid-experiment
recording_cutoff = int(len(filt_el)/sampling_rate)
if breach_rate >= max_breach_rate and secs_above_cutoff >= max_secs_above_cutoff and mean_breach_rate_persec >= max_mean_breach_rate_persec:
# Find the first 1 second epoch where the number of cutoff breaches is higher than the maximum allowed mean breach rate
recording_cutoff = np.where(breaches_per_sec > max_mean_breach_rate_persec)[0][0]
# Dump a plot showing where the recording was cut off at
fig = plt.figure()
plt.plot(np.arange(test_el.shape[0]), np.mean(test_el, axis = 1))
plt.plot((recording_cutoff, recording_cutoff), (np.min(np.mean(test_el, axis = 1)), np.max(np.mean(test_el, axis = 1))), 'k-', linewidth = 4.0)
plt.xlabel('Recording time (secs)', fontsize=18)
plt.ylabel('Average voltage recorded\nper sec (microvolts)', fontsize=18)
plt.title('Recording cutoff time\n(indicated by the black horizontal line)', fontsize=18)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
save_file = os.path.join(file_dir,'Plots',str(electrode_num),'Plots','cutoff_time.png')
fig.savefig(save_file, bbox_inches='tight')
plt.close("all")
# Then cut the recording accordingly
filt_el = filt_el[:recording_cutoff*int(sampling_rate)]
if len(filt_el)==0:
print('Immediate Cutoff for electrode %i...exiting' % electrode_num)
return electrode_num, 0, recording_cutoff
# Slice waveforms out of the filtered electrode recordings
slices, spike_times = clust.extract_waveforms(filt_el, spike_snapshot = [spike_snapshot_before, spike_snapshot_after], sampling_rate = sampling_rate)
if len(slices)==0:
print('No spikes found for electrode %i...exiting' % electrode_num)
return electrode_num, 0, recording_cutoff
# Delete filtered electrode from memory
del filt_el, test_el
# Dejitter these spike waveforms, and get their maximum amplitudes
slices_dejittered, times_dejittered = clust.dejitter(slices, spike_times, spike_snapshot = [spike_snapshot_before, spike_snapshot_after], sampling_rate = sampling_rate)
try:
amplitudes = np.min(slices_dejittered, axis = 1)
except:
# This error triggers if slices dejittered have different lengths and thus cannot cast into 2D numpy array
# Has been fixed in slices dejittered to throw out spikes whose minimum is
# too close to the end of the waveform to get a full snapshot, happens only
# for a few spikes since extract waveforms should make properly sized
# waveforms around min, but not always
print('Electrode %i slices_dejittered error: Dejitter shape %s & slices shape %s' % (electrode_num,str(slices_dejittered.shape),str(slices.shape)))
return electrode_num, -1, recording_cutoff
# Delete the original slices and times now that dejittering is complete
del slices; del spike_times
wave_dir = os.path.join(file_dir, 'spike_waveforms', 'electrode%i' % electrode_num)
time_dir = os.path.join(file_dir, 'spike_times', 'electrode%i' % electrode_num)
plot_dir = os.path.join(file_dir, 'Plots', '%i' % electrode_num, 'Plots')
clust_dir = os.path.join(file_dir, 'clustering_results', 'electrode%i' % electrode_num)
# Save these slices/spike waveforms and their times to their respective folders
np.save(os.path.join(wave_dir, 'spike_waveforms.npy'), slices_dejittered)
np.save(os.path.join(time_dir, 'spike_times.npy'), times_dejittered)
# Scale the dejittered slices by the energy of the waveforms
scaled_slices, energy = clust.scale_waveforms(slices_dejittered)
# Run PCA on the scaled waveforms
pca_slices, explained_variance_ratio = clust.implement_pca(scaled_slices)
# Save the pca_slices, energy and amplitudes to the spike_waveforms folder for this electrode
np.save(os.path.join(wave_dir, 'pca_waveforms.npy'), pca_slices)
np.save(os.path.join(wave_dir, 'energy.npy'), energy)
np.save(os.path.join(wave_dir, 'spike_amplitudes.npy'), amplitudes)
# Create file for saving plots, and plot explained variance ratios of the PCA
fig = plt.figure()
x = np.arange(len(explained_variance_ratio))
plt.plot(x, explained_variance_ratio)
plt.title('Variance ratios explained by PCs',fontsize=26)
plt.xlabel('PC #',fontsize=24)
plt.ylabel('Explained variance ratio',fontsize=24)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.savefig(os.path.join(plot_dir, 'pca_variance.png'), bbox_inches='tight')
plt.close("all")
# Make an array of the data to be used for clustering, and delete pca_slices, scaled_slices, energy and amplitudes
# Use trough to prior peak slope and the first 3 PCs
n_pc = 3
data = np.zeros((len(pca_slices), n_pc + 3))
data[:,3:] = pca_slices[:,:n_pc]
data[:,0] = energy[:]/np.max(energy)
data[:,1] = np.abs(amplitudes)/np.max(np.abs(amplitudes))
del pca_slices; del scaled_slices; del energy
# Get slopes
for i, wave in enumerate(slices_dejittered):
peaks = find_peaks(wave)[0]
minima = np.argmin(wave)
if not any(peaks < minima):
maxima = np.argmax(wave[:minima])
else:
maxima = max(peaks[np.where(peaks < minima)[0]])
data[i,2] = (wave[minima]-wave[maxima])/(minima-maxima)
# Run GMM, from 2 to max_clusters
for i in range(max_clusters-1):
try:
model, predictions, bic = clust.clusterGMM(data, n_clusters = i+2, n_iter = num_iter, restarts = num_restarts, threshold = thresh)
except:
#print "Clustering didn't work - solution with %i clusters most likely didn't converge" % (i+2)
continue
# Sometimes large amplitude noise waveforms cluster with the spike waveforms because the amplitude has been factored out of the scaled slices.
# Run through the clusters and find the waveforms that are more than wf_amplitude_sd_cutoff larger than the cluster mean. Set predictions = -1 at these points so that they aren't picked up by blech_post_process
for cluster in range(i+2):
cluster_points = np.where(predictions[:] == cluster)[0]
this_cluster = predictions[cluster_points]
cluster_amplitudes = amplitudes[cluster_points]
cluster_amplitude_mean = np.mean(cluster_amplitudes)
cluster_amplitude_sd = np.std(cluster_amplitudes)
reject_wf = np.where(cluster_amplitudes <= cluster_amplitude_mean - wf_amplitude_sd_cutoff*cluster_amplitude_sd)[0]
this_cluster[reject_wf] = -1
predictions[cluster_points] = this_cluster
# Make folder for results of i+2 clusters, and store results there
tmp_clust_dir = os.path.join(clust_dir, 'clusters%i' % (i+2))
os.mkdir(tmp_clust_dir)
np.save(os.path.join(tmp_clust_dir, 'predictions.npy'), predictions)
np.save(os.path.join(tmp_clust_dir, 'bic.npy'), bic)
# Plot the graphs, for this set of clusters, in the directory made for this electrode
tmp_plot_dir = os.path.join(plot_dir, '%i_clusters' % (i+2))
os.mkdir(tmp_plot_dir)
colors = cm.rainbow(np.linspace(0, 1, i+2))
for feature1 in range(len(data[0])):
for feature2 in range(len(data[0])):
if feature1 < feature2:
fig = plt.figure()
plt_names = []
for cluster in range(i+2):
plot_data = np.where(predictions[:] == cluster)[0]
plt_names.append(plt.scatter(data[plot_data, feature1], data[plot_data, feature2], color = colors[cluster], s = 0.8))
plt.xlabel("Feature %i" % feature1)
plt.ylabel("Feature %i" % feature2)
# Produce figure legend
plt.legend(tuple(plt_names), tuple("Cluster %i" % cluster for cluster in range(i+2)), scatterpoints = 1, loc = 'lower left', ncol = 3, fontsize = 8)
plt.title("%i clusters" % (i+2))
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.savefig(os.path.join(tmp_plot_dir, 'feature%ivs%i.png' % (feature2, feature1)))
plt.close("all")
for cluster in range(i+2):
fig = plt.figure()
cluster_points = np.where(predictions[:] == cluster)[0]
for other_cluster in range(i+2):
mahalanobis_dist = []
other_cluster_mean = model.means_[other_cluster, :]
other_cluster_covar_I = linalg.inv(model.covariances_[other_cluster, :, :])
for points in cluster_points:
mahalanobis_dist.append(mahalanobis(data[points, :], other_cluster_mean, other_cluster_covar_I))
# Plot histogram of Mahalanobis distances
y,binEdges=np.histogram(mahalanobis_dist)
bincenters = 0.5*(binEdges[1:] + binEdges[:-1])
plt.plot(bincenters, y, label = 'Dist from cluster %i' % other_cluster)
plt.xlabel('Mahalanobis distance')
plt.ylabel('Frequency')
plt.legend(loc = 'upper right', fontsize = 8)
plt.title('Mahalanobis distance of Cluster %i from all other clusters' % cluster, fontsize=24)
fig.savefig(os.path.join(tmp_plot_dir, 'Mahalonobis_cluster%i.png' % cluster))
plt.close("all")
# Create file, and plot spike waveforms for the different clusters. Plot 10 times downsampled dejittered/smoothed waveforms.
# Additionally plot the ISI distribution of each cluster
tmp_plot_dir = os.path.join(plot_dir, '%i_clusters_waveforms_ISIs' % (i+2))
os.mkdir(tmp_plot_dir)
for cluster in range(i+2):
cluster_points = np.where(predictions[:] == cluster)[0]
if cluster_points.shape[0] == 0:
print('No cluster points for %s: electrode %i, cluster %i'
% (os.path.basename(file_dir), electrode_num, cluster))
continue
fig, ax = blech_waveforms_datashader.waveforms_datashader(slices_dejittered[cluster_points, :], dir_name = "datashader_temp_el%i" % electrode_num)
ax.set_xlabel('Sample ({:d} samples per ms)'.format(int(sampling_rate/1000)), fontsize=20)
ax.set_ylabel('Voltage (microvolts)', fontsize=20)
ax.set_title('Cluster%i' % cluster, fontsize=26)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.savefig(os.path.join(tmp_plot_dir, 'Cluster%i_waveforms' % cluster))
plt.close("all")
fig = plt.figure()
cluster_times = times_dejittered[cluster_points]
ISIs = np.ediff1d(np.sort(cluster_times))
ISIs = ISIs/(sampling_rate/1000)
try:
plt.hist(ISIs, bins = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, np.max(ISIs)])
except:
print('Electrode %i clustering %i has no ISIs: %s' % (electrode_num,cluster,str(ISIs.shape)))
return electrode_num, -1, recording_cutoff
plt.xlim([0.0, 10.0])
plt.title("2ms ISI violations = %.1f percent (%i/%i)" %((float(len(np.where(ISIs < 2.0)[0]))/float(len(cluster_times)))*100.0, len(np.where(ISIs < 2.0)[0]), len(cluster_times)) + '\n' + "1ms ISI violations = %.1f percent (%i/%i)" %((float(len(np.where(ISIs < 1.0)[0]))/float(len(cluster_times)))*100.0, len(np.where(ISIs < 1.0)[0]), len(cluster_times)), fontsize=16)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
fig.savefig(os.path.join(tmp_plot_dir, 'Cluster%i_ISIs' % cluster))
plt.close("all")
# Make file for dumping info about memory usage
f = open(os.path.join(file_dir, 'memory_monitor_clustering', '%i.txt' % electrode_num), 'w')
print(mm.memory_usage_resource(), file=f)
f.close()
print('Finished clustering electrode %i' % electrode_num)
return electrode_num, 1, recording_cutoff
(electrode_num, i + 2, cluster))
plt.close("all")
# Create file, and plot spike waveforms for the different clusters. Plot 10 times downsampled dejittered/smoothed waveforms.
# Additionally plot the ISI distribution of each cluster
os.mkdir('./Plots/%i/Plots/%i_clusters_waveforms_ISIs' %
(electrode_num, i + 2))
for cluster in range(i + 2):
cluster_points = np.where(predictions[:] == cluster)[0]
if cluster_points.shape[0] == 0:
print('No cluster points for %s: electrode %i, cluster %i' %
(os.path.basename(file_dir), electrode_num, cluster))
continue
fig, ax = blech_waveforms_datashader.waveforms_datashader(
slices_dejittered[cluster_points, :],
dir_name="datashader_temp_el%i" % electrode_num)
ax.set_xlabel('Sample ({:d} samples per ms)'.format(
int(sampling_rate / 1000)))
ax.set_ylabel('Voltage (microvolts)')
ax.set_title('Cluster%i' % cluster)
fig.savefig(
'./Plots/%i/Plots/%i_clusters_waveforms_ISIs/Cluster%i_waveforms' %
(electrode_num, i + 2, cluster))
plt.close("all")
fig = plt.figure()
cluster_times = times_dejittered[cluster_points]
ISIs = np.ediff1d(np.sort(cluster_times))
ISIs = ISIs / (sampling_rate / 1000)
try: