def slice_clusters(comm, params, result, to_remove=[], to_merge=[], extension='', light=False):
import h5py, shutil
file_out_suff = params.get('data', 'file_out_suff')
N_e = params.getint('data', 'N_e')
if comm.rank == 0:
print_and_log(['Node 0 is slicing clusters'], 'debug', params)
if to_merge != []:
for count in xrange(len(to_merge)):
remove = to_merge[count][1]
to_remove += [remove]
all_elements = [[] for i in xrange(N_e)]
for target in numpy.unique(to_remove):
elec = result['electrodes'][target]
nic = target - numpy.where(result['electrodes'] == elec)[0][0]
mask = result['clusters_' + str(elec)] > -1
tmp = numpy.unique(result['clusters_' + str(elec)][mask])
all_elements[elec] += list(numpy.where(result['clusters_' + str(elec)] == tmp[nic])[0])
for elec in xrange(N_e):
if not light:
result['data_' + str(elec)] = numpy.delete(result['data_' + str(elec)], all_elements[elec], axis=0)
result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec])
result['times_' + str(elec)] = numpy.delete(result['times_' + str(elec)], all_elements[elec])
result['peaks_' + str(elec)] = numpy.delete(result['peaks_' + str(elec)], all_elements[elec])
else:
result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec])
myfile = h5py.File(file_out_suff + '.clusters.hdf5', 'r', libver='latest')
data = myfile.get('data_' + str(elec))[:]
result['data_' + str(elec)] = numpy.delete(data, all_elements[elec], axis=0)
data = myfile.get('times_' + str(elec))[:]
result['times_' + str(elec)] = numpy.delete(data, all_elements[elec])
data = myfile.get('peaks_' + str(elec))[:]
result['peaks_' + str(elec)] = numpy.delete(data, all_elements[elec])
myfile.close()
result['electrodes'] = numpy.delete(result['electrodes'], numpy.unique(to_remove))
cfile = h5py.File(file_out_suff + '.clusters-new.hdf5', 'w', libver='latest')
to_write = ['data_', 'clusters_', 'times_', 'peaks_']
for ielec in xrange(N_e):
write_datasets(cfile, to_write, result, ielec)
write_datasets(cfile, ['electrodes'], result)
cfile.close()
if os.path.exists(file_out_suff + '.clusters%s.hdf5' %extension):
os.remove(file_out_suff + '.clusters%s.hdf5' %extension)
shutil.move(file_out_suff + '.clusters-new.hdf5', file_out_suff + '.clusters%s.hdf5' %extension)
comm.Barrier()
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
gheader = Fore.GREEN + get_header()
header = gheader + Fore.RESET
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for slicing results',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = circus.shared.utils.io.load_parameters(filename)
file_out_suff = params.get('data', 'file_out_suff')
if not params.get('data', 'multi-files'):
print_and_log(['Not a multi-file!'], 'error', params)
sys.exit(0)
to_process = circus.shared.files.get_multi_files(params)
result = circus.shared.files.get_results(params, extension=extension)
times = circus.shared.files.data_stats(params,
show=False,
export_times=True)
sub_results = slice_result(result, times)
for count, result in enumerate(sub_results):
keys = ['spiketimes', 'amplitudes']
mydata = h5py.File(file_out_suff + '.result%s_%d.hdf5' %
(extension, count),
'w',
libver='latest')
for key in keys:
mydata.create_group(key)
for temp in result[key].keys():
tmp_path = '%s/%s' % (key, temp)
mydata.create_dataset(tmp_path, data=result[key][temp])
mydata.close()
def slice_templates(comm, params, to_remove=[], to_merge=[], extension=''):
import shutil, h5py
file_out_suff = params.get('data', 'file_out_suff')
if comm.rank == 0:
print_and_log(['Node 0 is slicing templates'], 'debug', params)
old_templates = load_data(params, 'templates')
old_limits = load_data(params, 'limits')
N_e = params.getint('data', 'N_e')
N_t = params.getint('data', 'N_t')
x, N_tm = old_templates.shape
norm_templates = load_data(params, 'norm-templates')
if to_merge != []:
for count in xrange(len(to_merge)):
remove = to_merge[count][1]
to_remove += [remove]
all_templates = set(numpy.arange(N_tm//2))
to_keep = numpy.array(list(all_templates.difference(to_remove)))
positions = numpy.arange(len(to_keep))
local_keep = to_keep[positions]
templates = scipy.sparse.lil_matrix((N_e*N_t, 2*len(to_keep)), dtype=numpy.float32)
hfile = h5py.File(file_out_suff + '.templates-new.hdf5', 'w', libver='latest')
norms = hfile.create_dataset('norms', shape=(2*len(to_keep), ), dtype=numpy.float32, chunks=True)
limits = hfile.create_dataset('limits', shape=(len(to_keep), 2), dtype=numpy.float32, chunks=True)
for count, keep in zip(positions, local_keep):
templates[:, count] = old_templates[:, keep]
templates[:, count + len(to_keep)] = old_templates[:, keep + N_tm//2]
norms[count] = norm_templates[keep]
norms[count + len(to_keep)] = norm_templates[keep + N_tm//2]
if to_merge == []:
new_limits = old_limits[keep]
else:
subset = numpy.where(to_merge[:, 0] == keep)[0]
if len(subset) > 0:
idx = numpy.unique(to_merge[subset].flatten())
ratios = norm_templates[idx]/norm_templates[keep]
new_limits = [numpy.min(ratios*old_limits[idx][:, 0]), numpy.max(ratios*old_limits[idx][:, 1])]
else:
new_limits = old_limits[keep]
limits[count] = new_limits
templates = templates.tocoo()
hfile.create_dataset('temp_x', data=templates.row)
hfile.create_dataset('temp_y', data=templates.col)
hfile.create_dataset('temp_data', data=templates.data)
hfile.create_dataset('temp_shape', data=numpy.array([N_e, N_t, 2*len(to_keep)], dtype=numpy.int32))
hfile.close()
if os.path.exists(file_out_suff + '.templates%s.hdf5' %extension):
os.remove(file_out_suff + '.templates%s.hdf5' %extension)
shutil.move(file_out_suff + '.templates-new.hdf5', file_out_suff + '.templates%s.hdf5' %extension)
comm.Barrier()
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
gheader = Fore.GREEN + get_header()
header = gheader + Fore.RESET
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = circus.shared.utils.io.load_parameters(filename)
sampling_rate = params.getint('data', 'sampling_rate')
data_dtype = params.get('data', 'data_dtype')
gain = 1
file_out_suff = params.get('data', 'file_out_suff')
data_offset = params.getint('data', 'data_offset')
probe = read_probe(params)
if extension != '':
extension = '-' + extension
def generate_matlab_mapping(probe):
p = {}
positions = []
nodes = []
for key in probe['channel_groups'].keys():
p.update(probe['channel_groups'][key]['geometry'])
nodes += probe['channel_groups'][key]['channels']
positions += [
p[channel]
for channel in probe['channel_groups'][key]['channels']
]
idx = numpy.argsort(nodes)
positions = numpy.array(positions)[idx]
t = tempfile.NamedTemporaryFile().name + '.hdf5'
cfile = h5py.File(t, 'w')
to_write = {
'positions': positions / 10.,
'permutation': numpy.sort(nodes),
'nb_total': numpy.array([probe['total_nb_channels']])
}
write_datasets(cfile, to_write.keys(), to_write)
cfile.close()
return t
mapping = generate_matlab_mapping(probe)
filename = params.get('data', 'data_file')
gui_params = [
sampling_rate,
os.path.abspath(file_out_suff),
'%s.mat' % extension, mapping, 2, data_dtype, data_offset, gain,
filename
]
gui_file = pkg_resources.resource_filename(
'circus', os.path.join('matlab_GUI', 'SortingGUI.m'))
# Change to the directory of the matlab file
os.chdir(os.path.abspath(os.path.dirname(gui_file)))
# Use quotation marks for string arguments
is_string = [False, True, True, True, False, True, False, False, True]
arguments = ', '.join([
"'%s'" % arg if s else "%s" % arg
for arg, s in zip(gui_params, is_string)
])
matlab_command = 'SortingGUI(%s)' % arguments
print_and_log(["Launching the MATLAB GUI..."], 'info', params)
try:
sys.exit(
subprocess.call(
['matlab', '-nodesktop', '-nosplash', '-r', matlab_command]))
except Exception:
print_error(
["Something wrong with MATLAB. Try circus-gui-python instead?"])
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
gheader = Fore.GREEN + get_header()
header = gheader + Fore.RESET
parser = argparse.ArgumentParser(description=header,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e', '--extension', help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = circus.shared.utils.io.load_parameters(filename)
mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(['You need to update phy-contrib to the latest git version'], 'error', params)
sys.exit(0)
sampling_rate = float(params.getint('data', 'sampling_rate'))
data_dtype = params.get('data', 'data_dtype')
file_out_suff = params.get('data', 'file_out_suff')
data_offset = params.getint('data', 'data_offset')
probe = read_probe(params)
if extension != '':
extension = '-' + extension
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(['Data should be first exported with the converting method!'], 'error', params)
else:
print_and_log(["Launching the phy GUI..."], 'info', params)
gui_params = {}
gui_params['dat_path'] = params.get('data', 'data_file')
gui_params['n_channels_dat'] = params.getint('data', 'N_total')
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = params.get('data', 'data_dtype')
gui_params['offset'] = params.getint('data', 'data_offset')
gui_params['sample_rate'] = params.getint('data', 'sampling_rate')
gui_params['hp_filtered'] = True
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
import h5py
parallel_hdf5 = h5py.get_config().mpi
from mpi4py import MPI
try:
SHARED_MEMORY = True
MPI.Win.Allocate_shared(1, 1, MPI.INFO_NULL, MPI.COMM_SELF).Free()
except (NotImplementedError, AttributeError):
SHARED_MEMORY = False
user_path = pjoin(os.path.expanduser('~'), 'spyking-circus')
tasks_list = None
if not os.path.exists(user_path):
os.makedirs(user_path)
try:
import cudamat as cmt
cmt.init()
HAVE_CUDA = True
except Exception:
HAVE_CUDA = False
all_steps = [
'whitening', 'clustering', 'fitting', 'gathering', 'extracting',
'filtering', 'converting', 'benchmarking', 'merging', 'validating'
]
if os.path.exists(user_path + 'config.params'):
config_file = os.path.abspath(user_path + 'config.params')
else:
config_file = os.path.abspath(
pkg_resources.resource_filename('circus', 'config.params'))
gheader = Fore.GREEN + get_header()
header = gheader
header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str(
psutil.cpu_count()) + '\n'
header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str(
HAVE_CUDA) + '\n'
header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str(
parallel_hdf5) + '\n'
header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str(
SHARED_MEMORY) + '\n'
header += '\n'
header += Fore.GREEN + "##################################################################"
header += Fore.RESET
method_help = '''by default, first 4 steps are performed,
but a subset x,y can be done. Steps are:
- filtering
- whitening
- clustering
- fitting
- (extra) merging [GUI for meta merging]
- (extra) converting [export results to phy format]
- (extra) gathering [force collection of results]
- (extra) extracting [get templates from spike times]
- (extra) benchmarking [with -o and -t]
- (extra) validating [to compare performance with GT neurons]'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile',
help='data file (or a list of commands if batch mode)')
parser.add_argument('-m',
'--method',
default='filtering,whitening,clustering,fitting',
help=method_help)
parser.add_argument('-c',
'--cpu',
type=int,
default=1,
help='number of CPU')
parser.add_argument('-g',
'--gpu',
type=int,
default=0,
help='number of GPU')
parser.add_argument('-H',
'--hostfile',
help='hostfile for MPI',
default=pjoin(user_path, 'circus.hosts'))
parser.add_argument(
'-b',
'--batch',
help='datafile is a list of commands to launch, in a batch mode',
action='store_true')
parser.add_argument(
'-p',
'--preview',
help='GUI to display the first second filtered with thresholds',
action='store_true')
parser.add_argument('-r',
'--result',
help='GUI to display the results on top of raw data',
action='store_true')
parser.add_argument(
'-e',
'--extension',
help='extension to consider for merging and converting',
default='None')
parser.add_argument(
'-o',
'--output',
help='output file [for generation of synthetic benchmarks]')
parser.add_argument('-t',
'--type',
help='benchmark type',
choices=['fitting', 'clustering', 'synchrony'])
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
steps = args.method.split(',')
for step in steps:
if step not in all_steps:
print_error(['The method "%s" is not recognized' % step])
sys.exit(1)
# To save some typing later
(nb_cpu, nb_gpu, hostfile, batch, preview, result, extension, output,
benchmark) = (args.cpu, args.gpu, args.hostfile, args.batch, args.preview,
args.result, args.extension, args.output, args.type)
filename = os.path.abspath(args.datafile)
f_next, extens = os.path.splitext(filename)
if extens == '.params':
print_error(['You should launch the code on the data file!'])
sys.exit(1)
file_params = f_next + '.params'
if not os.path.exists(file_params) and not batch:
print Fore.RED + 'The parameter file %s is not present!' % file_params
key = ''
while key not in ['y', 'n']:
key = raw_input(
Fore.WHITE +
"Do you want SpyKING CIRCUS to create a parameter file? [y/n]")
if key == 'y':
print Fore.WHITE + "Generating template file", file_params
print Fore.WHITE + "Fill it properly before launching the code! (see documentation)"
shutil.copyfile(config_file, file_params)
sys.exit()
elif batch:
tasks_list = filename
if not batch:
params = io.load_parameters(filename)
if preview:
print_info(
['Preview mode, showing only first second of the recording'])
tmp_path_loc = os.path.join(
os.path.abspath(params.get('data', 'data_file_noext')), 'tmp')
if not os.path.exists(tmp_path_loc):
os.makedirs(tmp_path_loc)
filename = os.path.join(tmp_path_loc, os.path.basename(filename))
f_next, extens = os.path.splitext(filename)
shutil.copyfile(file_params, f_next + '.params')
steps = ['filtering', 'whitening']
io.prepare_preview(params, filename)
io.change_flag(filename, 'chunk_size', '2')
io.change_flag(filename, 'safety_time', '0')
if tasks_list is not None:
with open(tasks_list, 'r') as f:
for line in f:
if len(line) > 0:
subprocess.check_call(['spyking-circus'] +
line.replace('\n', '').split(" "))
else:
if os.path.exists(f_next + '.log'):
os.remove(f_next + '.log')
write_to_logger(params, ['Config file: %s' % (f_next + '.params')],
'debug')
write_to_logger(params, ['Data file : %s' % filename], 'debug')
print gheader
if preview:
print Fore.GREEN + "Steps :", Fore.CYAN + "preview mode"
elif result:
print Fore.GREEN + "Steps :", Fore.CYAN + "results mode"
else:
print Fore.GREEN + "Steps :", Fore.CYAN + ", ".join(steps)
print Fore.GREEN + "GPU detected :", Fore.CYAN + str(HAVE_CUDA)
print Fore.GREEN + "Number of CPU :", Fore.CYAN + str(
nb_cpu) + "/" + str(psutil.cpu_count())
if HAVE_CUDA:
print Fore.GREEN + "Number of GPU :", Fore.CYAN + str(nb_gpu)
print Fore.GREEN + "Parallel HDF5 :", Fore.CYAN + str(parallel_hdf5)
print Fore.GREEN + "Shared memory :", Fore.CYAN + str(SHARED_MEMORY)
print Fore.GREEN + "Hostfile :", Fore.CYAN + hostfile
print ""
print Fore.GREEN + "##################################################################"
print ""
print Fore.RESET
# Launch the subtasks
subtasks = [('filtering', 'mpirun'), ('whitening', 'mpirun'),
('clustering', 'mpirun'), ('fitting', 'mpirun'),
('extracting', 'mpirun'), ('gathering', 'python'),
('converting', 'mpirun'), ('benchmarking', 'mpirun'),
('merging', 'mpirun'), ('validating', 'mpirun')]
if HAVE_CUDA and nb_gpu > 0:
use_gpu = 'True'
else:
use_gpu = 'False'
time = circus.shared.io.data_stats(params) / 60.
if nb_cpu < psutil.cpu_count():
if use_gpu != 'True' and not result:
io.print_and_log([
'Using only %d out of %d local CPUs available (-c to change)'
% (nb_cpu, psutil.cpu_count())
], 'info', params)
if params.getboolean('detection',
'matched-filter') and not params.getboolean(
'clustering', 'smart_search'):
io.print_and_log(
['Smart Search should be activated for matched filtering'],
'info', params)
if time > 30 and not params.getboolean('clustering', 'smart_search'):
io.print_and_log(
['Smart Search could be activated for long recordings'],
'info', params)
n_edges = circus.shared.io.get_averaged_n_edges(params)
if n_edges > 100 and not params.getboolean('clustering', 'compress'):
io.print_and_log([
'Template compression is highly recommended based on parameters'
], 'info', params)
if params.getint('data', 'N_e') > 500:
if (params.getint('data', 'chunk_size') > 10) or (params.getint(
'whitening', 'chunk_size') > 10):
io.print_and_log([
"Large number of electrodes, reduce chunk sizes to 10s in [data] and [whitening]"
], 'info', params)
if not result:
for subtask, command in subtasks:
if subtask in steps:
if command == 'python':
# Directly call the launcher
try:
circus.launch(subtask, filename, nb_cpu, nb_gpu,
use_gpu)
except:
print_error(['Step "%s" failed!' % subtask])
raise
elif command == 'mpirun':
# Use mpirun to make the call
mpi_args = gather_mpi_arguments(hostfile, params)
if subtask != 'fitting':
nb_tasks = str(max(args.cpu, args.gpu))
else:
if use_gpu == 'True':
nb_tasks = str(args.gpu)
else:
nb_tasks = str(args.cpu)
if subtask == 'benchmarking':
if (output is None) or (benchmark is None):
print_error([
"To generate synthetic datasets, you must provide output and type"
])
sys.exit()
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu, output, benchmark
]
elif subtask in ['merging', 'converting']:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu, extension
]
else:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu
]
write_to_logger(params,
['Launching task %s' % subtask],
'debug')
write_to_logger(params,
['Command: %s' % str(mpi_args)],
'debug')
try:
subprocess.check_call(mpi_args)
except:
print_error(['Step "%s" failed!' % subtask])
raise
if preview or result:
from circus.shared import gui
import pylab
from matplotlib.backends import qt_compat
use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE
if use_pyside:
from PySide import QtGui, QtCore, uic
else:
from PyQt4 import QtGui, QtCore, uic
app = QtGui.QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
if preview:
mygui = gui.PreviewGUI(io.load_parameters(filename))
shutil.rmtree(tmp_path_loc)
elif result:
mygui = gui.PreviewGUI(io.load_parameters(filename), show_fit=True)
sys.exit(app.exec_())
def main(filename, params, nb_cpu, nb_gpu, use_gpu, extension):
params = circus.shared.utils.io.load_parameters(filename)
sampling_rate = float(params.getint('data', 'sampling_rate'))
data_dtype = params.get('data', 'data_dtype')
file_out_suff = params.get('data', 'file_out_suff')
data_offset = params.getint('data', 'data_offset')
probe = read_probe(params)
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
N_e = params.getint('data', 'N_e')
N_t = params.getint('data', 'N_t')
erase_all = params.getboolean('converting', 'erase_all')
export_pcs = params.get('converting', 'export_pcs')
def generate_mapping(probe):
p = {}
positions = []
nodes = []
for key in probe['channel_groups'].keys():
p.update(probe['channel_groups'][key]['geometry'])
nodes += probe['channel_groups'][key]['channels']
positions += [
p[channel]
for channel in probe['channel_groups'][key]['channels']
]
idx = numpy.argsort(nodes)
positions = numpy.array(positions)[idx]
return positions
def get_max_loc_channel(params):
nodes, edges = circus.shared.utils.io.get_nodes_and_edges(params)
max_loc_channel = 0
for key in edges.keys():
if len(edges[key]) > max_loc_channel:
max_loc_channel = len(edges[key])
return max_loc_channel
def write_results(path, params, extension):
result = get_results(params, extension)
spikes = numpy.zeros(0, dtype=numpy.uint64)
clusters = numpy.zeros(0, dtype=numpy.uint32)
amplitudes = numpy.zeros(0, dtype=numpy.double)
for key in result['spiketimes'].keys():
temp_id = int(key.split('_')[-1])
data = result['spiketimes'].pop(key).astype(numpy.uint64)
spikes = numpy.concatenate((spikes, data))
data = result['amplitudes'].pop(key).astype(numpy.double)
amplitudes = numpy.concatenate((amplitudes, data[:, 0]))
clusters = numpy.concatenate(
(clusters,
temp_id * numpy.ones(len(data), dtype=numpy.uint32)))
idx = numpy.argsort(spikes)
numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx])
numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx])
numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx])
return
def write_templates(path, params, extension):
max_loc_channel = get_max_loc_channel(params)
templates = load_data(params, 'templates', extension)
N_tm = templates.shape[1] // 2
to_write = numpy.zeros((N_tm, N_t, N_e), dtype=numpy.float32)
mapping = numpy.zeros((N_tm, max_loc_channel), dtype=numpy.int32)
for t in xrange(N_tm):
tmp = templates[:, t].toarray().reshape(N_e, N_t).T
x, y = tmp.nonzero()
to_write[t, x, y] = tmp[x, y]
nb_loc = len(numpy.unique(y))
mapping[t, numpy.arange(nb_loc)] = numpy.unique(y)
numpy.save(os.path.join(output_path, 'templates'),
to_write.astype(numpy.single))
numpy.save(os.path.join(output_path, 'templates_ind'),
mapping.astype(numpy.double))
return N_tm
def write_pcs(path, params, comm, extension, mode=0):
spikes = numpy.load(os.path.join(output_path, 'spike_times.npy'))
labels = numpy.load(os.path.join(output_path, 'spike_templates.npy'))
max_loc_channel = get_max_loc_channel(params)
nb_features = params.getint('whitening', 'output_dim')
nodes, edges = get_nodes_and_edges(params)
N_total = params.getint('data', 'N_total')
templates = load_data(params, 'templates', extension)
N_tm = templates.shape[1] // 2
pc_features_ind = numpy.zeros((N_tm, max_loc_channel),
dtype=numpy.int32)
clusters = load_data(params, 'clusters', extension)
best_elec = clusters['electrodes']
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
for count, elec in enumerate(best_elec):
nb_loc = len(edges[nodes[elec]])
pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[
nodes[elec]]]
basis_proj, basis_rec = load_data(params, 'basis')
to_process = numpy.arange(comm.rank, N_tm, comm.size)
all_offsets = numpy.zeros(N_tm, dtype=numpy.int32)
for target in xrange(N_tm):
if mode == 0:
all_offsets[target] = len(numpy.where(labels == target)[0])
elif mode == 1:
all_offsets[target] = min(
500, len(numpy.where(labels == target)[0]))
all_paddings = numpy.concatenate(([0], numpy.cumsum(all_offsets)))
total_pcs = numpy.sum(all_offsets)
pc_file = os.path.join(output_path, 'pc_features.npy')
pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy')
from numpy.lib.format import open_memmap
if comm.rank == 0:
pc_features = open_memmap(pc_file,
shape=(total_pcs, nb_features,
max_loc_channel),
dtype=numpy.float32,
mode='w+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids,
shape=(total_pcs, ),
dtype=numpy.int32,
mode='w+')
comm.Barrier()
pc_features = open_memmap(pc_file, mode='r+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids, mode='r+')
if comm.rank == 0:
pbar = get_progressbar(len(to_process))
all_idx = numpy.zeros(0, dtype=numpy.int32)
for gcount, target in enumerate(to_process):
count = all_paddings[target]
if mode == 1:
idx = numpy.random.permutation(
numpy.where(labels == target)[0])[:500]
pc_ids[count:count + len(idx)] = idx
elif mode == 0:
idx = numpy.where(labels == target)[0]
elec = best_elec[target]
indices = inv_nodes[edges[nodes[elec]]]
labels_i = target * numpy.ones(len(idx))
times_i = numpy.take(spikes, idx)
sub_data = get_stas(params,
times_i,
labels_i,
elec,
neighs=indices,
nodes=nodes,
auto_align=False)
pcs = numpy.dot(sub_data, basis_proj)
pcs = numpy.swapaxes(pcs, 1, 2)
if mode == 0:
pc_features[idx, :, :len(indices)] = pcs
elif mode == 1:
pc_features[count:count + len(idx), :, :len(indices)] = pcs
if comm.rank == 0:
pbar.update(gcount)
if comm.rank == 0:
pbar.finish()
comm.Barrier()
if comm.rank == 0:
numpy.save(os.path.join(output_path, 'pc_feature_ind'),
pc_features_ind.astype(
numpy.uint32)) #n_templates, n_loc_chan
do_export = True
if comm.rank == 0:
if os.path.exists(output_path):
if not erase_all:
key = ''
while key not in ['y', 'n']:
print(
Fore.WHITE +
"Export already made! Do you want to erase everything? (y)es / (n)o "
)
key = raw_input('')
if key == 'y':
do_export = True
else:
do_export = False
if do_export:
if os.path.exists(os.path.abspath('.phy')):
shutil.rmtree(os.path.abspath('.phy'))
shutil.rmtree(output_path)
if do_export == True:
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
elif do_export == False:
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
else:
do_export = bool(
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0))
comm.Barrier()
if do_export:
if comm.rank == 0:
os.makedirs(output_path)
print_and_log(
["Exporting data for the phy GUI with %d CPUs..." % nb_cpu],
'info', params)
numpy.save(
os.path.join(output_path, 'whitening_mat'),
load_data(params, 'spatial_whitening').astype(numpy.double))
numpy.save(os.path.join(output_path, 'channel_positions'),
generate_mapping(probe).astype(numpy.double))
nodes, edges = get_nodes_and_edges(params)
numpy.save(os.path.join(output_path, 'channel_map'),
nodes.astype(numpy.int32))
write_results(output_path, params, extension)
N_tm = write_templates(output_path, params, extension)
similarities = h5py.File(file_out_suff +
'.templates%s.hdf5' % extension,
'r+',
libver='latest').get('maxoverlap')
norm = params.getint('data', 'N_e') * params.getint('data', 'N_t')
numpy.save(os.path.join(output_path, 'similar_templates'),
(similarities[:N_tm, :N_tm] / norm).astype(
numpy.single))
comm.Barrier()
make_pcs = 2
if comm.rank == 0:
if export_pcs == 'prompt':
key = ''
while key not in ['a', 's', 'n']:
print(
Fore.WHITE +
"Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o"
)
key = raw_input('')
else:
key = export_pcs
if key == 'a':
make_pcs = 0
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
elif key == 's':
make_pcs = 1
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
elif key == 'n':
comm.bcast(numpy.array([2], dtype=numpy.int32), root=0)
if os.path.exists(os.path.join(output_path,
'pc_features.npy')):
os.remove(os.path.join(output_path, 'pc_features.npy'))
if os.path.exists(
os.path.join(output_path, 'pc_feature_ind.npy')):
os.remove(os.path.join(output_path, 'pc_feature_ind.npy'))
else:
make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
make_pcs = make_pcs[0]
comm.Barrier()
if make_pcs < 2:
write_pcs(output_path, params, comm, extension, make_pcs)