def test_corrcoef_binned(self):
'''
Test the correlation coefficient between two binned spike trains.
'''
# Calculate clipped and unclipped
res_clipped = sc.correlation_coefficient(self.binned_st, binary=True)
res_unclipped = sc.correlation_coefficient(self.binned_st,
binary=False)
# Check dimensions
self.assertEqual(len(res_clipped), 2)
self.assertEqual(len(res_unclipped), 2)
# Check result unclipped against result calculated from scratch for
# the off-diagonal element
mat = self.binned_st.to_array()
mean_0 = np.mean(mat[0])
mean_1 = np.mean(mat[1])
target_from_scratch = \
np.dot(mat[0] - mean_0, mat[1] - mean_1) / \
np.sqrt(
np.dot(mat[0] - mean_0, mat[0] - mean_0) *
np.dot(mat[1] - mean_1, mat[1] - mean_1))
# Check result unclipped against result calculated by numpy.corrcoef
target_numpy = np.corrcoef(mat)
self.assertAlmostEqual(target_from_scratch, target_numpy[0][1])
self.assertAlmostEqual(res_unclipped[0][1], target_from_scratch)
self.assertAlmostEqual(res_unclipped[1][0], target_from_scratch)
# Check result clipped against result calculated from scratch for
# the off-diagonal elemant
mat = self.binned_st.to_bool_array()
mean_0 = np.mean(mat[0])
mean_1 = np.mean(mat[1])
target_from_scratch = \
np.dot(mat[0] - mean_0, mat[1] - mean_1) / \
np.sqrt(
np.dot(mat[0] - mean_0, mat[0] - mean_0) *
np.dot(mat[1] - mean_1, mat[1] - mean_1))
# Check result unclipped against result calculated by numpy.corrcoef
target_numpy = np.corrcoef(mat)
self.assertAlmostEqual(target_from_scratch, target_numpy[0][1])
self.assertAlmostEqual(res_clipped[0][1], target_from_scratch)
self.assertAlmostEqual(res_clipped[1][0], target_from_scratch)
def compute_spikes_correlation_coefficient(self,
binned_spikes_trains,
binsize=None,
num_bins=None,
**kwargs):
"""A method to compute the correlation coefficients
among the spikes' trains of a set of elephant.conversion.BinnedSpikeTrain,
using the elephant.spike_train_correlation.correlation_coefficient method.
- binned_spikes_trains: an elephant.conversion.BinnedSpikeTrain instance, or a
sequence (array, list, tuple) of Spikes Trains or of spikes' times arrays
- binsize: the size (float, in ms) of the bin to be used. Default=None.
- num_bins: the number (integer > 0) of bins to be used. Default=None.
If none of binsize or num_bins if given, a bin size equal to the sampling period is used.
Returns:
- a dictionary of the following key-value pair(s):
"correlation_coefficient": correlation_coefficient array
"binned_spikes_trains": the elephant.conversion.BinnedSpikeTrain instance used for the computation
"""
binned_spikes_trains = self._assert_binned_spikes_trains(
binned_spikes_trains, binsize, num_bins)
from elephant.spike_train_correlation import correlation_coefficient
return {
self._get_comput_res_type():
correlation_coefficient(binned_spikes_trains, **kwargs),
self.binned_spikes_trains_name:
binned_spikes_trains
}
def test_corrcoef_binned_short_input(self):
"""
Test if input list of one binned spike train yields 1.0.
"""
# Calculate correlation
binned_st = conv.BinnedSpikeTrain(
self.st_0, t_start=0 * pq.ms, t_stop=50. * pq.ms,
bin_size=1 * pq.ms)
result = sc.correlation_coefficient(binned_st, fast=False)
target = np.array(1.)
# Check result and dimensionality of result
self.assertEqual(result.ndim, 0)
assert_array_almost_equal(result, target)
assert_array_almost_equal(
result, sc.correlation_coefficient(
binned_st, fast=True))
def test_corrcoef_binned_same_spiketrains(self):
"""
Test if the correlation coefficient between two identical binned spike
trains evaluates to a 2x2 matrix of ones.
"""
# Calculate correlation
binned_st = conv.BinnedSpikeTrain(
[self.st_0, self.st_0], t_start=0 * pq.ms, t_stop=50. * pq.ms,
bin_size=1 * pq.ms)
result = sc.correlation_coefficient(binned_st, fast=False)
target = np.ones((2, 2))
# Check dimensions
self.assertEqual(len(result), 2)
# Check result
assert_array_almost_equal(result, target)
assert_array_almost_equal(
result, sc.correlation_coefficient(
binned_st, fast=True))
def test_empty_spike_train(self):
"""
Test whether a warning is yielded in case of empty spike train.
Also check correctness of the output array.
"""
# st_2 is empty
binned_12 = conv.BinnedSpikeTrain([self.st_1, self.st_2],
bin_size=1 * pq.ms)
with self.assertWarns(UserWarning):
result = sc.correlation_coefficient(binned_12, fast=False)
# test for NaNs in the output array
target = np.zeros((2, 2)) * np.NaN
target[0, 0] = 1.0
assert_array_almost_equal(result, target)
def test_cross_correlation_histogram(self):
"""
Test generic result of a cross-correlation histogram between two binned
spike trains.
"""
# Calculate CCH using Elephant (normal and binary version) with
# mode equal to 'full' (whole spike trains are correlated)
cch_clipped, bin_ids_clipped = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='full',
binary=True)
cch_unclipped, bin_ids_unclipped = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='full', binary=False)
cch_clipped_mem, bin_ids_clipped_mem = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='full',
binary=True, method='memory')
cch_unclipped_mem, bin_ids_unclipped_mem = \
sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='full',
binary=False, method='memory')
# Check consistency two methods
assert_array_equal(
np.squeeze(cch_clipped.magnitude), np.squeeze(
cch_clipped_mem.magnitude))
assert_array_equal(
np.squeeze(cch_clipped.times), np.squeeze(
cch_clipped_mem.times))
assert_array_equal(
np.squeeze(cch_unclipped.magnitude), np.squeeze(
cch_unclipped_mem.magnitude))
assert_array_equal(
np.squeeze(cch_unclipped.times), np.squeeze(
cch_unclipped_mem.times))
assert_array_almost_equal(bin_ids_clipped, bin_ids_clipped_mem)
assert_array_almost_equal(bin_ids_unclipped, bin_ids_unclipped_mem)
# Check normal correlation Note: Use numpy correlate to verify result.
# Note: numpy conventions for input array 1 and input array 2 are
# swapped compared to Elephant!
mat1 = self.binned_st1.to_array()[0]
mat2 = self.binned_st2.to_array()[0]
target_numpy = np.correlate(mat2, mat1, mode='full')
assert_array_equal(
target_numpy, np.squeeze(cch_unclipped.magnitude))
# Check cross correlation function for several displacements tau
# Note: Use Elephant corrcoeff to verify result
tau = [-25.0, 0.0, 13.0] # in ms
for t in tau:
# adjust t_start, t_stop to shift by tau
t0 = np.min([self.st_1.t_start + t * pq.ms, self.st_2.t_start])
t1 = np.max([self.st_1.t_stop + t * pq.ms, self.st_2.t_stop])
st1 = neo.SpikeTrain(self.st_1.magnitude + t, units='ms',
t_start=t0 * pq.ms, t_stop=t1 * pq.ms)
st2 = neo.SpikeTrain(self.st_2.magnitude, units='ms',
t_start=t0 * pq.ms, t_stop=t1 * pq.ms)
binned_sts = conv.BinnedSpikeTrain([st1, st2],
bin_size=1 * pq.ms,
t_start=t0 * pq.ms,
t_stop=t1 * pq.ms)
# caluclate corrcoef
corrcoef = sc.correlation_coefficient(binned_sts)[1, 0]
# expand t_stop to have two spike trains with same length as st1,
# st2
st1 = neo.SpikeTrain(self.st_1.magnitude, units='ms',
t_start=self.st_1.t_start,
t_stop=self.st_1.t_stop + np.abs(t) * pq.ms)
st2 = neo.SpikeTrain(self.st_2.magnitude, units='ms',
t_start=self.st_2.t_start,
t_stop=self.st_2.t_stop + np.abs(t) * pq.ms)
binned_st1 = conv.BinnedSpikeTrain(
st1, t_start=0 * pq.ms, t_stop=(50 + np.abs(t)) * pq.ms,
bin_size=1 * pq.ms)
binned_st2 = conv.BinnedSpikeTrain(
st2, t_start=0 * pq.ms, t_stop=(50 + np.abs(t)) * pq.ms,
bin_size=1 * pq.ms)
# calculate CCHcoef and take value at t=tau
CCHcoef, _ = sc.cch(binned_st1, binned_st2,
cross_correlation_coefficient=True)
left_edge = - binned_st1.n_bins + 1
tau_bin = int(t / float(binned_st1.bin_size.magnitude))
assert_array_almost_equal(
corrcoef, CCHcoef[tau_bin - left_edge].magnitude)
# Check correlation using binary spike trains
mat1 = np.array(self.binned_st1.to_bool_array()[0], dtype=int)
mat2 = np.array(self.binned_st2.to_bool_array()[0], dtype=int)
target_numpy = np.correlate(mat2, mat1, mode='full')
assert_array_equal(
target_numpy, np.squeeze(cch_clipped.magnitude))
# Check the time axis and bin IDs of the resulting AnalogSignal
assert_array_almost_equal(
(bin_ids_clipped - 0.5) * self.binned_st1.bin_size,
cch_unclipped.times)
assert_array_almost_equal(
(bin_ids_clipped - 0.5) * self.binned_st1.bin_size,
cch_clipped.times)
# Calculate CCH using Elephant (normal and binary version) with
# mode equal to 'valid' (only completely overlapping intervals of the
# spike trains are correlated)
cch_clipped, bin_ids_clipped = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='valid',
binary=True)
cch_unclipped, bin_ids_unclipped = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='valid',
binary=False)
cch_clipped_mem, bin_ids_clipped_mem = sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='valid',
binary=True, method='memory')
cch_unclipped_mem, bin_ids_unclipped_mem = \
sc.cross_correlation_histogram(
self.binned_st1, self.binned_st2, window='valid',
binary=False, method='memory')
# Check consistency two methods
assert_array_equal(
np.squeeze(cch_clipped.magnitude), np.squeeze(
cch_clipped_mem.magnitude))
assert_array_equal(
np.squeeze(cch_clipped.times), np.squeeze(
cch_clipped_mem.times))
assert_array_equal(
np.squeeze(cch_unclipped.magnitude), np.squeeze(
cch_unclipped_mem.magnitude))
assert_array_equal(
np.squeeze(cch_unclipped.times), np.squeeze(
cch_unclipped_mem.times))
assert_array_equal(bin_ids_clipped, bin_ids_clipped_mem)
assert_array_equal(bin_ids_unclipped, bin_ids_unclipped_mem)
# Check normal correlation Note: Use numpy correlate to verify result.
# Note: numpy conventions for input array 1 and input array 2 are
# swapped compared to Elephant!
mat1 = self.binned_st1.to_array()[0]
mat2 = self.binned_st2.to_array()[0]
target_numpy = np.correlate(mat2, mat1, mode='valid')
assert_array_equal(
target_numpy, np.squeeze(cch_unclipped.magnitude))
# Check correlation using binary spike trains
mat1 = np.array(self.binned_st1.to_bool_array()[0], dtype=int)
mat2 = np.array(self.binned_st2.to_bool_array()[0], dtype=int)
target_numpy = np.correlate(mat2, mat1, mode='valid')
assert_array_equal(
target_numpy, np.squeeze(cch_clipped.magnitude))
# Check the time axis and bin IDs of the resulting AnalogSignal
assert_array_equal(
(bin_ids_clipped - 0.5) * self.binned_st1.bin_size,
cch_unclipped.times)
assert_array_equal(
(bin_ids_clipped - 0.5) * self.binned_st1.bin_size,
cch_clipped.times)
# Check for wrong window parameter setting
self.assertRaises(
ValueError, sc.cross_correlation_histogram, self.binned_st1,
self.binned_st2, window='dsaij')
self.assertRaises(
ValueError, sc.cross_correlation_histogram, self.binned_st1,
self.binned_st2, window='dsaij', method='memory')
def get_firing_rate_metrics(neuronset, spikes_fn, num_neurons=8000.,
rows=50000000., start_time=100., dt=1.,
window=1000., snapshot_dt=200000.,
isi_enabled=True, std_enabled=True,
cc_enabled=True):
"""Get various metrics from raster spike files.
:neuronset: name of neuron set being looked at
:spikes_fn: file name of spikes file
:num_neurons: number of neurons in neuron set
:rows: rows to be read in each pandas chunk
:start_time: time to start the processing at (ms)
:dt: increment value (ms)
:window: window to count spikes in (ms)
:snapshot_dt: interval between snapshots for ISI and STD metrics (ms)
:isi_enabled: if ISI CVs should be calculated
:std_enabled: if STD of firing rate should be calculated
:cc_enabled: if average spike correlation coefficient should be enabled
:returns: True if everything went OK, else False
"""
# Initial indices
left = 0.
right = 0.
num_neurons = int(num_neurons)
current_time = start_time
old_neuronIDs = numpy.array([])
old_times = numpy.array([])
lgr.info("Processing {}.".format(spikes_fn))
if not os.path.exists(spikes_fn):
lgr.error("File not found {}".format(spikes_fn))
return False
with open("mean-firing-rates-{}.gdf".format(neuronset), 'w') as fh1, \
open("std-firing-rates-{}.gdf".format(neuronset), 'w') as fh2, \
open("ISI-cv-{}.gdf".format(neuronset), 'w') as fh3, \
open("cc-{}.gdf".format(neuronset), 'w') as fh4:
for chunk in pandas.read_csv(spikes_fn, sep='\s+', # noqa: W605
names=["neuronID",
"spike_time"],
dtype={'neuronID': numpy.uint16,
'spike_time': float},
lineterminator="\n",
skipinitialspace=True,
header=None, index_col=None,
skip_blank_lines=True,
chunksize=rows):
# Drop rows with nan
chunk = chunk.dropna(how='any')
if not validate_raster_df(chunk):
lgr.error("Error in {}. Skipping.".format(spikes_fn))
return False
neuronIDs = numpy.array(chunk.values[:, 0])
times = numpy.array(chunk.values[:, 1])
# 200 neuronIDs per second = 2 neuronIDs per 0.01 second (dt) per
# neuron this implies 2 * 10000 neuronIDs for 10000 neurons need
# to be kept to make sure I have a proper sliding window of
# chunks
if len(old_neuronIDs) > 0:
neuronIDs = numpy.append(old_neuronIDs, neuronIDs)
times = numpy.append(old_times, times)
lgr.debug(
"Times from {} to {} being analysed containing {} rows".format(
times[0], times[-1], len(times)))
lgr.debug("Current time is {}".format(current_time))
# Reset chunks
left = 0
right = 0
while (current_time < math.floor(times[-1])):
# Initialise these to 0
mean_firing_rate = 0.
spikesnum = 0.
mystd = -1
left += numpy.searchsorted(times[left:],
(current_time - window),
side='left')
right = left + numpy.searchsorted(
times[left:], current_time,
side='right')
# point is lesser than the first value in the chunk
if right == 0 and left == 0:
lgr.warning("Point too small for chunk")
current_time = times[0]
lgr.warning("Time to reset to: {}".format(times[0]))
continue
# interval not found, no spikes - not necessarily at max
# the max check is in the while condition, and that
# ascertains if a new chunk should be read
if right == left:
lgr.warning("No spikes in interval at {}".format(
current_time))
# Increment it by snapshot_dt to ensure that the next check
# for ISI and STD metrics can be made.
# Ideally, I should be able to move it to times[left], but
# there is no guarantee that it would remain dividible by
# snapshot_dt. That would mean that the next bits are never
# run, even if there are spikes.
current_time += snapshot_dt
lgr.warning("Current time updated to {}".format(
current_time))
# Print NA values for STD and ISI which will not be
# calculated for this time
lgr.warning("Printing invalid values for STD and ISI CV")
# For gnuplot, lines starting with # are ignored.
# To skip these points and have a discontinuous graph in
# gnuplot, one must leave a blank line in the text.
print(
"#{}\tNA\n".format(current_time / 1000.),
file=fh2, flush=True)
print(
"#{}\tNA\n".format(current_time / 1000.),
file=fh3, flush=True)
continue
# could even just do right - left if all I'm using is len
thiswindow_neuronIDs = neuronIDs[left:right]
thiswindow_times = times[left:right]
# mean firing rate
spikesnum = float(len(thiswindow_neuronIDs))
mean_firing_rate = (spikesnum / num_neurons) / (window / 1000)
# total neuronIDs by number of neurons
print(
"{}\t{}".format(current_time / 1000.,
mean_firing_rate),
file=fh1, flush=True)
# We only get here if there are some spikes, so there's no need
# to check for that again.
# STD of firing rates and ISI cv - it just takes way too much
# time to do for each dt - my post processing wont finish. So,
# we calculate it at intervals
if ((current_time - start_time) % snapshot_dt == 0):
if std_enabled:
lgr.debug("STD for {}".format(
current_time))
# STD of firing rates
# calculate firing rates of each neuron in the window
# then find STD
spike_counts = collections.Counter(thiswindow_neuronIDs)
firing_rates = []
for neuron, count in spike_counts.items():
firing_rates.append(count / (window / 1000))
neurons_spiking = len(firing_rates)
# Add 0s for neurons that did not spike
for i in range(0, (num_neurons - neurons_spiking)):
firing_rates.append(0)
lgr.debug("std being calculated from {} values".format(
len(firing_rates)))
mystd = numpy.std(firing_rates)
print(
"{}\t{}".format(current_time / 1000., mystd),
file=fh2, flush=True)
if cc_enabled:
lgr.debug("CC for {}".format(
current_time))
# CC
# I do not sort them.
# Get unique neuron list
neurons = list(set(thiswindow_neuronIDs))
# Shuffle them so that the spike trains are from a shuffled
# pack of neurons when the CC is calculated
random.shuffle(neurons)
# Select at least 800 neurons
if len(neurons) > 800:
N = max(int(0.1 * len(neurons)), 800)
else:
N = len(neurons)
lgr.debug("CC is using {} neurons".format(N))
neurons = neurons[0:N]
spike_trains = []
# Get spike trains for each neuron
for nrn in list(neurons):
indices = [i for i, x in
enumerate(thiswindow_neuronIDs) if x == nrn]
nrn_spike_times = thiswindow_times[indices]
nrn_spiketrain = SpikeTrain(nrn_spike_times * ms,
t_stop=thiswindow_times[-1])
spike_trains.append(nrn_spiketrain)
bin_size = (5 * ms)
binned_spike_trains = BinnedSpikeTrain(spike_trains,
bin_size)
cc_matrix = correlation_coefficient(binned_spike_trains)
# elements in triangle: (N * (N-1)/2)
# mean of cc values = (sum of triangle)/(N * (N-1)/2)
avg_cc = (
numpy.nansum(numpy.tril(cc_matrix, -1)) / (N * (N - 1) / 2)
)
print(
"{}\t{}\t{}".format(current_time / 1000., N, avg_cc), file=fh4,
flush=True)
if isi_enabled:
# ISI stats
neurons = set(thiswindow_neuronIDs)
lgr.debug("ISI: {} neurons being analysed.".format(
len(neurons)))
# for all neurons in this window
ISI_cvs = []
for neuron in list(neurons):
indices = [i for i, x
in enumerate(thiswindow_neuronIDs)
if x == neuron]
neuron_times = [thiswindow_times[i] for i in
indices]
ISIs = []
if len(neuron_times) > 1:
# otherwise ISI is undefined in this window for
# this neuron
prev = neuron_times[0]
# get a list of ISIs
for neuron_time in neuron_times:
ISIs.append(neuron_time - prev)
prev = neuron_time
# for this neuron, get stats
ISI_mean = numpy.mean(ISIs)
ISI_std = numpy.std(ISIs)
ISI_cv = ISI_std / ISI_mean
if not numpy.isnan(ISI_cv):
ISI_cvs.append(ISI_cv)
print(
"{}\t{}".format(current_time / 1000.,
numpy.mean(ISI_cvs)),
file=fh3, flush=True)
current_time += dt
lgr.debug("Printed till {}".format(current_time))
old_times = numpy.array(times[(left - len(times)):])
old_neuronIDs = numpy.array(neuronIDs[(left - len(neuronIDs)):])
del neuronIDs
del times
gc.collect()
lgr.info("Finished processing {}".format(spikes_fn))
return True
