def write_firings_file(channels, times, labels, fname):
L = len(channels)
X = np.zeros((3, L), dtype='float64')
X[0, :] = channels
X[1, :] = times
X[2, :] = labels
mdaio.writemda64(X, fname)
def test_compute_templates():
M, N, K, T, L = 5, 1000, 6, 50, 100
X = np.random.rand(M, N)
mdaio.writemda32(X, 'tmp.mda')
F = np.zeros((3, L))
F[1, :] = 1 + np.random.randint(N, size=(1, L))
F[2, :] = 1 + np.random.randint(K, size=(1, L))
mdaio.writemda64(F, 'tmp2.mda')
ret = compute_templates(timeseries='tmp.mda',
firings='tmp2.mda',
templates_out='tmp3.mda',
clip_size=T)
assert (ret)
templates0 = mdaio.readmda('tmp3.mda')
assert (templates0.shape == (M, T, K))
return True
def create_label_map(*,
metrics,
label_map_out,
firing_rate_thresh=.05,
isolation_thresh=0.95,
noise_overlap_thresh=.03,
peak_snr_thresh=1.5):
"""
Generate a label map based on the metrics file, where labels being mapped to zero are to be removed.
Parameters
----------
metrics : INPUT
Path of metrics json file to be used for generating the label map
label_map_out : OUTPUT
Path to mda file where the second column is the present label, and the first column is the new label
...
firing_rate_thresh : float64
(Optional) firing rate must be above this
isolation_thresh : float64
(Optional) isolation must be above this
noise_overlap_thresh : float64
(Optional) noise_overlap_thresh must be below this
peak_snr_thresh : float64
(Optional) peak snr must be above this
"""
#TODO: Way to pass in logic or thresholds flexibly
label_map = []
#Load json
with open(metrics) as metrics_json:
metrics_data = json.load(metrics_json)
#Iterate through all clusters
for idx in range(len(metrics_data['clusters'])):
if metrics_data['clusters'][idx]['metrics']['firing_rate'] < firing_rate_thresh or \
metrics_data['clusters'][idx]['metrics']['isolation'] < isolation_thresh or \
metrics_data['clusters'][idx]['metrics']['noise_overlap'] > noise_overlap_thresh or \
metrics_data['clusters'][idx]['metrics']['peak_snr'] < peak_snr_thresh:
#Map to zero (mask out)
label_map.append([0, metrics_data['clusters'][idx]['label']])
elif metrics_data['clusters'][idx]['metrics'][
'bursting_parent']: #Check if burst parent exists
label_map.append([
metrics_data['clusters'][idx]['metrics']['bursting_parent'],
metrics_data['clusters'][idx]['label']
])
else:
label_map.append([
metrics_data['clusters'][idx]['label'],
metrics_data['clusters'][idx]['label']
]) # otherwise, map to itself!
#Writeout
return mdaio.writemda64(np.array(label_map), label_map_out)
def synthesize_random_firings(*,
firings_out,
K=20,
samplerate=30000,
duration=60):
"""
Synthesize random waveforms for use in creating a synthetic timeseries dataset
Parameters
----------
firings_out : OUTPUT
Path to output firings mda file. 3xL, L is the number of events, second row are timestamps, third row are integer unit labels
K : int
(Optional) number of simulated units
samplerate : double
(Optional) sampling frequency in Hz
duration : double
(Optional) duration of the simulated acquisition in seconds
"""
firing_rates = 3 * np.ones((K))
refr = 4
N = np.int64(duration * samplerate)
# events/sec * sec/timepoint * N
populations = np.ceil(firing_rates / samplerate * N).astype('int')
times = np.zeros(0)
labels = np.zeros(0)
for k in range(1, K + 1):
refr_timepoints = refr / 1000 * samplerate
times0 = np.random.rand(populations[k - 1]) * (N - 1) + 1
## make an interesting autocorrelogram shape
times0 = np.hstack(
(times0, times0 +
rand_distr2(refr_timepoints, refr_timepoints * 20, times0.size)))
times0 = times0[np.random.choice(times0.size, int(times0.size / 2))]
times0 = times0[np.where((0 <= times0) & (times0 < N))]
times0 = enforce_refractory_period(times0, refr_timepoints)
times = np.hstack((times, times0))
labels = np.hstack((labels, k * np.ones(times0.shape)))
sort_inds = np.argsort(times)
times = times[sort_inds]
labels = labels[sort_inds]
firings = np.zeros((3, times.size), dtype=np.float64)
firings[1, :] = times
firings[2, :] = labels
return mdaio.writemda64(firings, firings_out)
def apply_label_map(*, firings, label_map, firings_out):
"""
Apply a label map to a given firings, including masking and merging
Parameters
----------
firings : INPUT
Path of input firings mda file
label_map : INPUT
Path of input label map mda file [base 1, mapping to zero removes from firings]
firings_out : OUTPUT
...
"""
firings = mdaio.readmda(firings)
label_map = mdaio.readmda(label_map)
label_map = np.reshape(label_map, (-1, 2))
label_map = label_map[np.argsort(label_map[:,
0])] # Assure input is sorted
#Propagate merge pairs to lowest label number
for idx, label in enumerate(label_map[:, 1]):
# jfm changed on 12/8/17 because isin() is not isin() older versions of numpy. :)
#label_map[np.isin(label_map[:,0],label),0] = label_map[idx,0] # Input should be sorted
label_map[np.where(label_map[:, 0] == label)[0],
0] = label_map[idx, 0] # Input should be sorted
#Apply label map
for label_pair in range(label_map.shape[0]):
# jfm changed on 12/8/17 because isin() is not isin() older versions of numpy. :)
#firings[2, np.isin(firings[2, :], label_map[label_pair, 1])] = label_map[label_pair,0]
firings[2, np.where(
firings[2, :] == label_map[label_pair,
1])[0]] = label_map[label_pair, 0]
#Mask out all labels mapped to zero
firings = firings[:, firings[2, :] != 0]
#Write remapped firings
return mdaio.writemda64(firings, firings_out)
def run(self):
tmpdir=os.environ.get('ML_PROCESSOR_TEMPDIR')
if not tmpdir:
raise Exception('Environment variable not set: ML_PROCESSOR_TEMPDIR')
source_dir=os.path.dirname(os.path.realpath(__file__))
## todo: link rather than copy
print('Copying timeseries file: {} -> {}'.format(self.timeseries,tmpdir+'/raw.mda'))
copyfile(self.timeseries,tmpdir+'/raw.mda')
print('Reading timeseries header...')
HH=mdaio.readmda_header(tmpdir+'/raw.mda')
num_channels=HH.dims[0]
num_timepoints=HH.dims[1]
duration_minutes=num_timepoints/self.samplerate/60
print('Num. channels = {}, Num. timepoints = {}, duration = {} minutes'.format(num_channels,num_timepoints,duration_minutes))
print('Creating .prb file...')
prb_text=self._read_text_file(source_dir+'/template.prb')
prb_text=prb_text.replace('$num_channels$','{}'.format(num_channels))
prb_text=prb_text.replace('$radius$','{}'.format(self.adjacency_radius))
geom=np.genfromtxt(self.geom, delimiter=',')
geom_str='{\n'
for m in range(geom.shape[0]):
geom_str+=' {}: [{},{}],\n'.format(m,geom[m,0],geom[m,1]) # todo: handle 3d geom
geom_str+='}'
prb_text=prb_text.replace('$geometry$','{}'.format(geom_str))
self._write_text_file(tmpdir+'/geometry.prb',prb_text)
print('Creating .params file...')
txt=self._read_text_file(source_dir+'/template.params')
txt=txt.replace('$header_size$','{}'.format(HH.header_size))
txt=txt.replace('$prb_file$',tmpdir+'/geometry.prb')
txt=txt.replace('$dtype$',HH.dt)
txt=txt.replace('$num_channels$','{}'.format(num_channels))
txt=txt.replace('$samplerate$','{}'.format(self.samplerate))
txt=txt.replace('$template_width_ms$','{}'.format(self.template_width_ms))
txt=txt.replace('$spike_thresh$','{}'.format(self.spike_thresh))
if self.detect_sign>0:
peaks_str='positive'
elif self.detect_sign<0:
peaks_str='negative'
else:
peaks_str='both'
txt=txt.replace('$peaks$',peaks_str)
txt=txt.replace('$whitening_max_elts$','{}'.format(self.whitening_max_elts))
txt=txt.replace('$clustering_max_elts$','{}'.format(self.clustering_max_elts))
self._write_text_file(tmpdir+'/raw.params',txt)
print('Running spyking circus...')
#num_threads=np.maximum(1,int(os.cpu_count()/2))
num_threads=1 # for some reason, using more than 1 thread causes an error
cmd='spyking-circus {} -c {} '.format(tmpdir+'/raw.mda',num_threads)
print(cmd)
retcode=self._run_command_and_print_output(cmd)
if retcode != 0:
raise Exception('Spyking circus returned a non-zero exit code')
result_fname=tmpdir+'/raw/raw.result.hdf5'
if not os.path.exists(result_fname):
raise Exception('Result file does not exist: '+result_fname)
firings=sc_results_to_firings(result_fname)
print(firings.shape)
mdaio.writemda64(firings,self.firings_out)
return True
def separateSpikesInEpochs(data_dir=None,
firings_file='firings.curated.mda',
timestamp_files=None,
write_separated_spikes=True):
"""
Takes curated spikes from MountainSort and combines this information with spike timestamps to create separate curated spikes for each epoch
:firings_file: Curated firings file
:timestamp_files: Spike timestamps file list
:write_separated_spikes: If the separated spikes should be written back to the data directory.
:returns: List of spikes for each epoch
"""
if data_dir is None:
# Get the firings file
data_dir = QtHelperUtils.get_directory(
message="Select Curated firings location")
separated_tetrodes = []
curated_firings = []
merged_curated_firings = []
tetrode_list = os.listdir(data_dir)
for tt_dir in tetrode_list:
try:
if firings_file in os.listdir(data_dir + '/' + tt_dir):
curated_firings.append([])
separated_tetrodes.append(tt_dir)
firings_file_location = '/'.join(
[data_dir, tt_dir, firings_file])
merged_curated_firings.append(
mdaio.readmda(firings_file_location))
print(MODULE_IDENTIFIER +
'Read merged firings file for tetrode %s!' % tt_dir)
else:
print(MODULE_IDENTIFIER +
'Merged firings %s not found for tetrode %s!' %
(firings_file, tt_dir))
except (FileNotFoundError, IOError) as err:
print(MODULE_IDENTIFIER +
'Unable to read merged firings file for tetrode %s!' %
tt_dir)
print(err)
gui_root = Tk()
gui_root.withdraw()
timestamp_headers = []
if timestamp_files is None:
# Read all the timestamp files
timestamp_files = filedialog.askopenfilenames(initialdir=DEFAULT_SEARCH_PATH, \
title="Select all timestamp files", \
filetypes=(("Timestamps", ".mda"), ("All Files", "*.*")))
gui_root.destroy()
for ts_file in timestamp_files:
timestamp_headers.append(mdaio.readmda_header(ts_file))
# Now that we have both the timestamp headers and the timestamp files, we
# can separate spikes out. It is important here for the timestamp files to
# be in the same order as the curated firings as that is the only way for
# us to tell that the firings are being split up correctly.
print(MODULE_IDENTIFIER + 'Looking at spike timestamps in order')
print(timestamp_files)
# First splice up curated spikes into indiviual epochs
for tt_idx, tt_firings in enumerate(merged_curated_firings):
for ep_idx, ts_header in enumerate(timestamp_headers):
if tt_firings is None:
curated_firings[tt_idx].append(None)
continue
n_data_points = ts_header.dims[0]
print(MODULE_IDENTIFIER + 'Epoch ' + str(ep_idx) + ': ' +
str(n_data_points) + ' samples.')
last_spike_from_epoch = np.searchsorted(
tt_firings[1], n_data_points, side='left') - 1
# If there are no spikes in this epoch, there might still be some in future epochs!
if last_spike_from_epoch < 0:
tt_firings[1] = tt_firings[1] - float(n_data_points)
curated_firings[tt_idx].append(None)
continue
last_spike_sample_number = tt_firings[1][last_spike_from_epoch]
print(MODULE_IDENTIFIER + separated_tetrodes[tt_idx] + ': First spike ' + str(tt_firings[1][0])\
+ ', Last spike ' + str(last_spike_sample_number))
epoch_spikes = tt_firings[:, :last_spike_from_epoch]
curated_firings[tt_idx].append(epoch_spikes)
if last_spike_from_epoch < (len(tt_firings[1]) - 1):
# Slice the merged curated firing to only have the remaining spikes
tt_firings = tt_firings[:, last_spike_from_epoch + 1:]
print(MODULE_IDENTIFIER +
'Trimming curated spikes. Aggregate sample start ' +
str(tt_firings[1][0]))
tt_firings[1] = tt_firings[1] - float(n_data_points)
print(MODULE_IDENTIFIER + 'Sample number trimmed to ' +
str(tt_firings[1][0]))
else:
print(MODULE_IDENTIFIER + separated_tetrodes[tt_idx] +
', Reached end of curated firings at Epoch ' +
str(ep_idx))
tt_firings = None
print(MODULE_IDENTIFIER +
'Spikes separated in epochs. Substituting timestamps!')
# For each epoch replace the sample numbers with the corresponding
# timestamps. We are going through multiple revisions for this so that we
# only have to load one timestamp file at a time
for ep_idx, ts_file in enumerate(timestamp_files):
epoch_timestamps = mdaio.readmda(ts_file)
print(MODULE_IDENTIFIER + 'Epoch ' + str(ep_idx))
for tt_idx, tt_curated_firings in enumerate(curated_firings):
if tt_curated_firings[ep_idx] is None:
continue
# Need to use the original array because changes to tt_curated_firings do not get copied back
curated_firings[tt_idx][ep_idx][1] = epoch_timestamps[np.array(
tt_curated_firings[ep_idx][1], dtype=int)]
print(MODULE_IDENTIFIER + separated_tetrodes[tt_idx] + ': Samples (' + \
str(tt_curated_firings[ep_idx][1][0]) + ', ' + str(tt_curated_firings[ep_idx][1][-1]), ')')
if write_separated_spikes:
try:
for tt_idx, tet in enumerate(separated_tetrodes):
for ep_idx in range(len(timestamp_files)):
if curated_firings[tt_idx][ep_idx] is not None:
ep_firings_file_name = data_dir + '/' + tet + '/firings-' + \
str(ep_idx+1) + '.curated.mda'
mdaio.writemda64(curated_firings[tt_idx][ep_idx],
ep_firings_file_name)
except OSError as exception:
if exception.errno != errno.EEXIST:
print(MODULE_IDENTIFIER +
'Unable to write timestamped firings!')
print(exception)
return curated_firings