def main(params, nb_cpu, nb_gpu, use_gpu, extension):
logger = init_logging(params.logfile)
logger = logging.getLogger('circus.merging')
file_out_suff = params.get('data', 'file_out_suff')
extension_in = extension
extension_out = '-merged'
if comm.rank == 0:
if (extension !=
'') and (os.path.exists(file_out_suff +
'.result%s.hdf5' % extension_out)):
erase = query_yes_no(
"Export already made! Do you want to erase everything?",
default=None)
if erase:
purge(file_out_suff, extension)
extension_in = ''
comm.Barrier()
if comm.rank == 0:
app = QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
else:
app = None
if comm.rank == 0:
print_and_log(['Launching the merging GUI...'], 'debug', logger)
mygui = gui.MergeWindow(params, app, extension_in, extension_out)
sys.exit(app.exec_())
def main(params, nb_cpu, nb_gpu, use_gpu, extension):
_ = init_logging(params.logfile)
logger = logging.getLogger('circus.merging')
file_out_suff = params.get('data', 'file_out_suff')
erase_all = params.getboolean('merging', 'erase_all')
extension_in = extension
extension_out = '-merged'
# Erase previous results (if user agrees).
if comm.rank == 0:
existing_file_paths = [
file_path for file_path in [
file_out_suff + ".%s%s.hdf5" % (file_id, extension_out)
for file_id in ['templates', 'clusters', 'result']
] if os.path.isfile(file_path)
]
existing_directory_path = [
directory_path
for directory_path in [file_out_suff + "%s.GUI" % extension_out]
if os.path.isdir(directory_path)
]
if len(existing_file_paths) > 0 or len(existing_directory_path) > 0:
if not erase_all:
erase = query_yes_no(
"Merging already done! Do you want to erase previous merging results?",
default=None)
else:
erase = True
if erase:
for path in existing_file_paths:
os.remove(path)
if comm.rank == 0:
print_and_log(["Removed file %s" % path], 'debug',
logger)
for path in existing_directory_path:
shutil.rmtree(path)
if comm.rank == 0:
print_and_log(["Removed directory %s" % path], 'debug',
logger)
comm.Barrier()
if comm.rank == 0 and params.getfloat('merging', 'auto_mode') == 0:
app = QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
else:
app = None
if comm.rank == 0:
print_and_log(['Launching the merging GUI...'], 'debug', logger)
_ = gui.MergeWindow(params, app, extension_in, extension_out)
sys.exit(app.exec_())
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to launch the phy GUI and visualize the results.
[data must be first converted with the converting mode]
'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
if extension != '':
extension = '-' + extension
try:
import traitlets
except ImportError:
print_and_log(
['The package traitlets required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import click
except ImportError:
print_and_log(['The package click required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import joblib
except ImportError:
print_and_log(['The package joblib required by phy is not installed'],
'error', logger)
sys.exit(1)
if HAVE_PHYCONTRIB:
mytest = StrictVersion(
phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(
['You need to update phy-contrib to the latest git version'],
'error', logger)
sys.exit(1)
print_and_log([
'phy-contrib is deprecated, you should upgrade to phy 2.0 and phylib'
], 'info', logger)
if HAVE_PHYLIB:
try:
import colorcet
except ImportError:
print_and_log(
['The package colorcet required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import qtconsole
except ImportError:
print_and_log(
['The package qtconsole required by phy is not installed'],
'error', logger)
sys.exit(1)
if not test_patch_for_similarities(params, extension):
print_and_log(
['You should re-export the data because of a fix in 0.6'], 'error',
logger)
continue_anyway = query_yes_no(
Fore.WHITE + "Continue anyway (results may not be fully correct)?",
default=None)
if not continue_anyway:
sys.exit(1)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
if data_file.params.has_key('data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
file_format = data_file.description
file_out_suff = params.get('data', 'file_out_suff')
if file_format not in supported_by_phy:
print_and_log([
"File format %s is not supported by phy. TraceView disabled" %
file_format
], 'info', logger)
if numpy.iterable(data_file.gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
else:
if data_file.gain != 1:
print_and_log([
"Gain of %g is not supported by phy. Expecting a scaling mismatch"
% data_file.gain
], 'info', logger)
gain = data_file.gain
probe = params.probe
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(
['Data should be first exported with the converting method!'],
'error', logger)
else:
print_and_log(["Launching the phy GUI..."], 'info', logger)
gui_params = {}
if file_format in supported_by_phy:
if not params.getboolean('data', 'overwrite'):
gui_params['dat_path'] = r"%s" % params.get(
'data', 'data_file_no_overwrite')
else:
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True,
has_been_created=False)
gui_params['dat_path'] = [
r"%s" % f for f in data_file.get_file_names()
]
else:
gui_params['dat_path'] = r"%s" % params.get(
'data', 'data_file')
else:
gui_params['dat_path'] = 'giverandomname.dat'
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_dtype
gui_params['offset'] = data_offset
gui_params['sample_rate'] = params.rate
gui_params['dir_path'] = output_path
gui_params['hp_filtered'] = True
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(params, nb_cpu, nb_gpu, use_gpu, extension):
logger = init_logging(params.logfile)
logger = logging.getLogger('circus.converting')
data_file = params.data_file
file_out_suff = params.get('data', 'file_out_suff')
probe = params.probe
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
N_e = params.getint('data', 'N_e')
N_t = params.getint('detection', 'N_t')
erase_all = params.getboolean('converting', 'erase_all')
export_pcs = params.get('converting', 'export_pcs')
export_all = params.getboolean('converting', 'export_all')
sparse_export = params.getboolean('converting', 'sparse_export')
if export_all and not params.getboolean('fitting', 'collect_all'):
if comm.rank == 0:
print_and_log(['Export unfitted spikes only if [fitting] collect_all is True'], 'error', logger)
sys.exit(0)
def generate_mapping(probe):
p = {}
positions = []
nodes = []
for key in probe['channel_groups'].keys():
p.update(probe['channel_groups'][key]['geometry'])
nodes += probe['channel_groups'][key]['channels']
positions += [p[channel] for channel in probe['channel_groups'][key]['channels']]
idx = numpy.argsort(nodes)
positions = numpy.array(positions)[idx]
return positions
def get_max_loc_channel(params):
nodes, edges = get_nodes_and_edges(params)
max_loc_channel = 0
for key in edges.keys():
if len(edges[key]) > max_loc_channel:
max_loc_channel = len(edges[key])
return max_loc_channel
def write_results(path, params, extension):
result = io.get_results(params, extension)
spikes = numpy.zeros(0, dtype=numpy.uint64)
clusters = numpy.zeros(0, dtype=numpy.uint32)
amplitudes = numpy.zeros(0, dtype=numpy.double)
N_tm = len(result['spiketimes'])
for key in result['spiketimes'].keys():
temp_id = int(key.split('_')[-1])
data = result['spiketimes'].pop(key).astype(numpy.uint64)
spikes = numpy.concatenate((spikes, data))
data = result['amplitudes'].pop(key).astype(numpy.double)
amplitudes = numpy.concatenate((amplitudes, data[:, 0]))
clusters = numpy.concatenate((clusters, temp_id*numpy.ones(len(data), dtype=numpy.uint32)))
if export_all:
print_and_log(["Last %d templates are unfitted spikes on all electrodes" %N_e], 'info', logger)
garbage = io.load_data(params, 'garbage', extension)
for key in garbage['gspikes'].keys():
elec_id = int(key.split('_')[-1])
data = garbage['gspikes'].pop(key).astype(numpy.uint64)
spikes = numpy.concatenate((spikes, data))
amplitudes = numpy.concatenate((amplitudes, numpy.zeros(len(data))))
clusters = numpy.concatenate((clusters, (elec_id + N_tm)*numpy.ones(len(data), dtype=numpy.uint32)))
idx = numpy.argsort(spikes)
numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx])
numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx])
numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx])
return
def write_templates(path, params, extension):
max_loc_channel = get_max_loc_channel(params)
templates = io.load_data(params, 'templates', extension)
N_tm = templates.shape[1]//2
if sparse_export:
n_channels_max = 0
for t in xrange(N_tm):
data = numpy.sum(numpy.sum(templates[:, t].toarray().reshape(N_e, N_t), 1) != 0)
if data > n_channels_max:
n_channels_max = data
else:
n_channels_max = N_e
if export_all:
to_write_sparse = numpy.zeros((N_tm + N_e, N_t, n_channels_max), dtype=numpy.float32)
mapping_sparse = -1 * numpy.ones((N_tm + N_e, n_channels_max), dtype=numpy.int32)
else:
to_write_sparse = numpy.zeros((N_tm, N_t, n_channels_max), dtype=numpy.float32)
mapping_sparse = -1 * numpy.ones((N_tm, n_channels_max), dtype=numpy.int32)
for t in xrange(N_tm):
tmp = templates[:, t].toarray().reshape(N_e, N_t).T
x, y = tmp.nonzero()
nb_loc = len(numpy.unique(y))
if sparse_export:
all_positions = numpy.zeros(y.max()+1, dtype=numpy.int32)
all_positions[numpy.unique(y)] = numpy.arange(nb_loc, dtype=numpy.int32)
pos = all_positions[y]
to_write_sparse[t, x, pos] = tmp[x, y]
mapping_sparse[t, numpy.arange(nb_loc)] = numpy.unique(y)
else:
pos = y
to_write_sparse[t, x, pos] = tmp[x, y]
if export_all:
for t in xrange(N_tm, N_tm + N_e):
mapping_sparse[t, 0] = t - N_tm
numpy.save(os.path.join(output_path, 'templates'), to_write_sparse)
if sparse_export:
numpy.save(os.path.join(output_path, 'template_ind'), mapping_sparse)
return N_tm
def write_pcs(path, params, extension, N_tm, mode=0):
spikes = numpy.load(os.path.join(output_path, 'spike_times.npy'))
labels = numpy.load(os.path.join(output_path, 'spike_templates.npy'))
max_loc_channel = get_max_loc_channel(params)
nb_features = params.getint('whitening', 'output_dim')
sign_peaks = params.get('detection', 'peaks')
nodes, edges = get_nodes_and_edges(params)
N_total = params.getint('data', 'N_total')
if export_all:
nb_templates = N_tm + N_e
else:
nb_templates = N_tm
pc_features_ind = numpy.zeros((nb_templates, max_loc_channel), dtype=numpy.int32)
clusters = io.load_data(params, 'clusters', extension)
best_elec = clusters['electrodes']
if export_all:
best_elec = numpy.concatenate((best_elec, numpy.arange(N_e)))
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
for count, elec in enumerate(best_elec):
nb_loc = len(edges[nodes[elec]])
pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[nodes[elec]]]
if sign_peaks in ['negative', 'both']:
basis_proj, basis_rec = io.load_data(params, 'basis')
elif sign_peaks in ['positive']:
basis_proj, basis_rec = io.load_data(params, 'basis-pos')
to_process = numpy.arange(comm.rank, nb_templates, comm.size)
all_offsets = numpy.zeros(nb_templates, dtype=numpy.int32)
for target in xrange(nb_templates):
if mode == 0:
all_offsets[target] = len(numpy.where(labels == target)[0])
elif mode == 1:
all_offsets[target] = min(500, len(numpy.where(labels == target)[0]))
all_paddings = numpy.concatenate(([0] , numpy.cumsum(all_offsets)))
total_pcs = numpy.sum(all_offsets)
pc_file = os.path.join(output_path, 'pc_features.npy')
pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy')
from numpy.lib.format import open_memmap
if comm.rank == 0:
pc_features = open_memmap(pc_file, shape=(total_pcs, nb_features, max_loc_channel), dtype=numpy.float32, mode='w+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids, shape=(total_pcs, ), dtype=numpy.int32, mode='w+')
comm.Barrier()
pc_features = open_memmap(pc_file, mode='r+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids, mode='r+')
to_explore = xrange(comm.rank, nb_templates, comm.size)
if comm.rank == 0:
to_explore = get_tqdm_progressbar(to_explore)
all_idx = numpy.zeros(0, dtype=numpy.int32)
for gcount, target in enumerate(to_explore):
count = all_paddings[target]
if mode == 1:
idx = numpy.random.permutation(numpy.where(labels == target)[0])[:500]
pc_ids[count:count+len(idx)] = idx
elif mode == 0:
idx = numpy.where(labels == target)[0]
elec = best_elec[target]
indices = inv_nodes[edges[nodes[elec]]]
labels_i = target*numpy.ones(len(idx))
times_i = numpy.take(spikes, idx).astype(numpy.int64)
sub_data = io.get_stas(params, times_i, labels_i, elec, neighs=indices, nodes=nodes, auto_align=False)
pcs = numpy.dot(sub_data, basis_proj)
pcs = numpy.swapaxes(pcs, 1,2)
if mode == 0:
pc_features[idx, :, :len(indices)] = pcs
elif mode == 1:
pc_features[count:count+len(idx), :, :len(indices)] = pcs
comm.Barrier()
if comm.rank == 0:
numpy.save(os.path.join(output_path, 'pc_feature_ind'), pc_features_ind.astype(numpy.uint32)) #n_templates, n_loc_chan
do_export = True
if comm.rank == 0:
if os.path.exists(output_path):
if not erase_all:
do_export = query_yes_no(Fore.WHITE + "Export already made! Do you want to erase everything?", default=None)
if do_export:
if os.path.exists(os.path.abspath('.phy')):
shutil.rmtree(os.path.abspath('.phy'))
shutil.rmtree(output_path)
if do_export == True:
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
elif do_export == False:
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
else:
do_export = bool(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0))
comm.Barrier()
if do_export:
if comm.rank == 0:
os.makedirs(output_path)
print_and_log(["Exporting data for the phy GUI with %d CPUs..." %nb_cpu], 'info', logger)
if params.getboolean('whitening', 'spatial'):
whitening_mat = io.load_data(params, 'spatial_whitening').astype(numpy.double)
numpy.save(os.path.join(output_path, 'whitening_mat'), whitening_mat)
numpy.save(os.path.join(output_path, 'whitening_mat_inv'), numpy.linalg.inv(whitening_mat))
else:
numpy.save(os.path.join(output_path, 'whitening_mat'), numpy.eye(N_e))
numpy.save(os.path.join(output_path, 'channel_positions'), generate_mapping(probe).astype(numpy.double))
nodes, edges = get_nodes_and_edges(params)
numpy.save(os.path.join(output_path, 'channel_map'), nodes.astype(numpy.int32))
write_results(output_path, params, extension)
N_tm = write_templates(output_path, params, extension)
apply_patch_for_similarities(params, extension)
template_file = h5py.File(file_out_suff + '.templates%s.hdf5' %extension, 'r', libver='earliest')
similarities = template_file.get('maxoverlap')[:]
template_file.close()
norm = N_e*N_t
if export_all:
to_write = numpy.zeros((N_tm + N_e, N_tm + N_e), dtype=numpy.single)
to_write[:N_tm, :N_tm] = (similarities[:N_tm, :N_tm]/norm).astype(numpy.single)
else:
to_write = (similarities[:N_tm, :N_tm]/norm).astype(numpy.single)
numpy.save(os.path.join(output_path, 'similar_templates'), to_write)
comm.bcast(numpy.array([N_tm], dtype=numpy.int32), root=0)
else:
N_tm = int(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0))
comm.Barrier()
make_pcs = 2
if comm.rank == 0:
if export_pcs == 'prompt':
key = ''
while key not in ['a', 's', 'n']:
print(Fore.WHITE + "Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o")
key = raw_input('')
else:
key = export_pcs
if key == 'a':
make_pcs = 0
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
elif key == 's':
make_pcs = 1
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
elif key == 'n':
comm.bcast(numpy.array([2], dtype=numpy.int32), root=0)
if os.path.exists(os.path.join(output_path, 'pc_features.npy')):
os.remove(os.path.join(output_path, 'pc_features.npy'))
if os.path.exists(os.path.join(output_path, 'pc_feature_ind.npy')):
os.remove(os.path.join(output_path, 'pc_feature_ind.npy'))
else:
make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
make_pcs = make_pcs[0]
comm.Barrier()
if make_pcs < 2:
write_pcs(output_path, params, extension, N_tm, make_pcs)
def main(params, nb_cpu, nb_gpu, use_gpu, extension):
_ = init_logging(params.logfile)
logger = logging.getLogger('circus.converting')
data_file = params.data_file
file_out_suff = params.get('data', 'file_out_suff')
probe = params.probe
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
N_e = params.getint('data', 'N_e')
prelabelling = params.getboolean('converting', 'prelabelling')
N_t = params.getint('detection', 'N_t')
erase_all = params.getboolean('converting', 'erase_all')
export_pcs = params.get('converting', 'export_pcs')
export_all = params.getboolean('converting', 'export_all')
sparse_export = params.getboolean('converting', 'sparse_export')
rpv_threshold = params.getfloat('converting', 'rpv_threshold')
if export_all and not params.getboolean('fitting', 'collect_all'):
if comm.rank == 0:
print_and_log([
'Export unfitted spikes only if [fitting] collect_all is True'
], 'error', logger)
sys.exit(0)
def generate_mapping(probe):
p = {}
positions = []
nodes = []
shanks = []
for key in list(probe['channel_groups'].keys()):
p.update(probe['channel_groups'][key]['geometry'])
nodes += probe['channel_groups'][key]['channels']
positions += [
p[channel]
for channel in probe['channel_groups'][key]['channels']
]
shanks += [key] * len(probe['channel_groups'][key]['channels'])
positions = numpy.array(positions)
shanks = numpy.array(shanks)
return positions, shanks
def get_max_loc_channel(params, extension):
if test_if_support(params, extension):
supports = io.load_data(params, 'supports', extension)
max_loc_channel = numpy.sum(supports, 1).max()
else:
nodes, edges = get_nodes_and_edges(params)
max_loc_channel = 0
for key in list(edges.keys()):
if len(edges[key]) > max_loc_channel:
max_loc_channel = len(edges[key])
return max_loc_channel
def write_results(path, params, extension):
result = io.get_results(params, extension)
spikes = [numpy.zeros(0, dtype=numpy.uint64)]
clusters = [numpy.zeros(0, dtype=numpy.uint32)]
amplitudes = [numpy.zeros(0, dtype=numpy.double)]
N_tm = len(result['spiketimes'])
has_purity = test_if_purity(params, extension)
rpvs = []
if prelabelling:
labels = []
norms = io.load_data(params, 'norm-templates', extension)
norms = norms[:len(norms) // 2]
if has_purity:
purity = io.load_data(params, 'purity', extension)
for key in list(result['spiketimes'].keys()):
temp_id = int(key.split('_')[-1])
myspikes = result['spiketimes'].pop(key).astype(numpy.uint64)
spikes.append(myspikes)
myamplitudes = result['amplitudes'].pop(key).astype(numpy.double)
amplitudes.append(myamplitudes[:, 0])
clusters.append(temp_id *
numpy.ones(len(myamplitudes), dtype=numpy.uint32))
rpv = get_rpv(myspikes, params.data_file.sampling_rate)
rpvs += [[temp_id, rpv]]
if prelabelling:
if has_purity:
if rpv <= rpv_threshold:
if purity[temp_id] > 0.75:
labels += [[temp_id, 'good']]
else:
if purity[temp_id] > 0.75:
labels += [[temp_id, 'mua']]
else:
labels += [[temp_id, 'noise']]
else:
median_amp = numpy.median(myamplitudes[:, 0])
std_amp = numpy.std(myamplitudes[:, 0])
if rpv <= rpv_threshold and numpy.abs(median_amp -
1) < 0.25:
labels += [[temp_id, 'good']]
else:
if median_amp < 0.5:
labels += [[temp_id, 'mua']]
elif norms[temp_id] < 0.1:
labels += [[temp_id, 'noise']]
if export_all:
print_and_log([
"Last %d templates are unfitted spikes on all electrodes" % N_e
], 'info', logger)
garbage = io.load_data(params, 'garbage', extension)
for key in list(garbage['gspikes'].keys()):
elec_id = int(key.split('_')[-1])
data = garbage['gspikes'].pop(key).astype(numpy.uint64)
spikes.append(data)
amplitudes.append(numpy.ones(len(data)))
clusters.append((elec_id + N_tm) *
numpy.ones(len(data), dtype=numpy.uint32))
if prelabelling:
f = open(os.path.join(output_path, 'cluster_group.tsv'), 'w')
f.write('cluster_id\tgroup\n')
for l in labels:
f.write('%s\t%s\n' % (l[0], l[1]))
f.close()
# f = open(os.path.join(output_path, 'cluster_rpv.tsv'), 'w')
# f.write('cluster_id\trpv\n')
# for l in rpvs:
# f.write('%s\t%s\n' % (l[0], l[1]))
# f.close()
spikes = numpy.concatenate(spikes).astype(numpy.uint64)
amplitudes = numpy.concatenate(amplitudes).astype(numpy.double)
clusters = numpy.concatenate(clusters).astype(numpy.uint32)
idx = numpy.argsort(spikes)
numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx])
numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx])
numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx])
return
def write_templates(path, params, extension):
max_loc_channel = get_max_loc_channel(params, extension)
templates = io.load_data(params, 'templates', extension)
N_tm = templates.shape[1] // 2
nodes, edges = get_nodes_and_edges(params)
if sparse_export:
n_channels_max = 0
for t in range(N_tm):
data = numpy.sum(
numpy.sum(templates[:, t].toarray().reshape(N_e, N_t), 1)
!= 0)
if data > n_channels_max:
n_channels_max = data
else:
n_channels_max = N_e
if export_all:
to_write_sparse = numpy.zeros((N_tm + N_e, N_t, n_channels_max),
dtype=numpy.float32)
mapping_sparse = -1 * numpy.ones(
(N_tm + N_e, n_channels_max), dtype=numpy.int32)
else:
to_write_sparse = numpy.zeros((N_tm, N_t, n_channels_max),
dtype=numpy.float32)
mapping_sparse = -1 * numpy.ones(
(N_tm, n_channels_max), dtype=numpy.int32)
has_purity = test_if_purity(params, extension)
if has_purity:
purity = io.load_data(params, 'purity', extension)
f = open(os.path.join(output_path, 'cluster_purity.tsv'), 'w')
f.write('cluster_id\tpurity\n')
for i in range(N_tm):
f.write('%d\t%g\n' % (i, purity[i]))
f.close()
for t in range(N_tm):
tmp = templates[:, t].toarray().reshape(N_e, N_t).T
x, y = tmp.nonzero()
nb_loc = len(numpy.unique(y))
if sparse_export:
all_positions = numpy.zeros(y.max() + 1, dtype=numpy.int32)
all_positions[numpy.unique(y)] = numpy.arange(
nb_loc, dtype=numpy.int32)
pos = all_positions[y]
to_write_sparse[t, x, pos] = tmp[x, y]
mapping_sparse[t, numpy.arange(nb_loc)] = numpy.unique(y)
else:
pos = y
to_write_sparse[t, x, pos] = tmp[x, y]
if export_all:
garbage = io.load_data(params, 'garbage', extension)
for t in range(N_tm, N_tm + N_e):
elec = t - N_tm
spikes = garbage['gspikes'].pop('elec_%d' % elec).astype(
numpy.int64)
spikes = numpy.random.permutation(spikes)[:100]
mapping_sparse[t, 0] = t - N_tm
waveform = io.get_stas(params,
times_i=spikes,
labels_i=np.ones(len(spikes)),
src=elec,
neighs=[elec],
nodes=nodes,
mean_mode=True)
nb_loc = 1
if sparse_export:
to_write_sparse[t, :, 0] = waveform
else:
to_write_sparse[t, :, elec] = waveform
numpy.save(os.path.join(output_path, 'templates'), to_write_sparse)
if sparse_export:
numpy.save(os.path.join(output_path, 'template_ind'),
mapping_sparse)
return N_tm
def write_pcs(path, params, extension, N_tm, mode=0):
spikes = numpy.load(os.path.join(output_path, 'spike_times.npy'))
labels = numpy.load(os.path.join(output_path, 'spike_templates.npy'))
max_loc_channel = get_max_loc_channel(params, extension)
nb_features = params.getint('whitening', 'output_dim')
sign_peaks = params.get('detection', 'peaks')
nodes, edges = get_nodes_and_edges(params)
N_total = params.getint('data', 'N_total')
has_support = test_if_support(params, extension)
if has_support:
supports = io.load_data(params, 'supports', extension)
else:
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.arange(len(nodes))
if export_all:
nb_templates = N_tm + N_e
else:
nb_templates = N_tm
pc_features_ind = numpy.zeros((nb_templates, max_loc_channel),
dtype=numpy.int32)
best_elec = io.load_data(params, 'electrodes', extension)
if export_all:
best_elec = numpy.concatenate((best_elec, numpy.arange(N_e)))
if has_support:
for count, support in enumerate(supports):
nb_loc = numpy.sum(support)
pc_features_ind[count, numpy.arange(nb_loc)] = numpy.where(
support == True)[0]
else:
for count, elec in enumerate(best_elec):
nb_loc = len(edges[nodes[elec]])
pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[
nodes[elec]]]
if sign_peaks in ['negative', 'both']:
basis_proj, basis_rec = io.load_data(params, 'basis')
elif sign_peaks in ['positive']:
basis_proj, basis_rec = io.load_data(params, 'basis-pos')
to_process = numpy.arange(comm.rank, nb_templates, comm.size)
all_offsets = numpy.zeros(nb_templates, dtype=numpy.int32)
for target in range(nb_templates):
if mode == 0:
all_offsets[target] = len(numpy.where(labels == target)[0])
elif mode == 1:
all_offsets[target] = min(
500, len(numpy.where(labels == target)[0]))
all_paddings = numpy.concatenate(([0], numpy.cumsum(all_offsets)))
total_pcs = numpy.sum(all_offsets)
pc_file = os.path.join(output_path, 'pc_features.npy')
pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy')
from numpy.lib.format import open_memmap
if comm.rank == 0:
pc_features = open_memmap(pc_file,
shape=(total_pcs, nb_features,
max_loc_channel),
dtype=numpy.float32,
mode='w+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids,
shape=(total_pcs, ),
dtype=numpy.int32,
mode='w+')
comm.Barrier()
pc_features = open_memmap(pc_file, mode='r+')
if mode == 1:
pc_ids = open_memmap(pc_file_ids, mode='r+')
to_explore = list(range(comm.rank, nb_templates, comm.size))
if comm.rank == 0:
to_explore = get_tqdm_progressbar(params, to_explore)
all_idx = numpy.zeros(0, dtype=numpy.int32)
for gcount, target in enumerate(to_explore):
count = all_paddings[target]
if mode == 1:
idx = numpy.random.permutation(
numpy.where(labels == target)[0])[:500]
pc_ids[count:count + len(idx)] = idx
elif mode == 0:
idx = numpy.where(labels == target)[0]
elec = best_elec[target]
if has_support:
if target >= len(supports):
indices = [target - N_tm]
else:
indices = numpy.where(supports[target])[0]
else:
indices = inv_nodes[edges[nodes[elec]]]
labels_i = target * numpy.ones(len(idx))
times_i = numpy.take(spikes, idx).astype(numpy.int64)
sub_data = io.get_stas(params,
times_i,
labels_i,
elec,
neighs=indices,
nodes=nodes,
auto_align=False)
pcs = numpy.dot(sub_data, basis_proj)
pcs = numpy.swapaxes(pcs, 1, 2)
if mode == 0:
pc_features[idx, :, :len(indices)] = pcs
elif mode == 1:
pc_features[count:count + len(idx), :, :len(indices)] = pcs
comm.Barrier()
if comm.rank == 0:
numpy.save(os.path.join(output_path, 'pc_feature_ind'),
pc_features_ind.astype(
numpy.uint32)) # n_templates, n_loc_chan
do_export = True
if comm.rank == 0:
if os.path.exists(output_path):
if not erase_all:
do_export = query_yes_no(
Fore.WHITE +
"Export already made! Do you want to erase everything?",
default=None)
if do_export:
if os.path.exists(os.path.abspath('.phy')):
shutil.rmtree(os.path.abspath('.phy'))
shutil.rmtree(output_path)
if do_export:
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
else:
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
else:
do_export = bool(
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0))
comm.Barrier()
if do_export:
#apply_patch_for_similarities(params, extension)
if comm.rank == 0:
os.makedirs(output_path)
print_and_log(
["Exporting data for the phy GUI with %d CPUs..." % nb_cpu],
'info', logger)
if params.getboolean('whitening', 'spatial'):
whitening_mat = io.load_data(
params, 'spatial_whitening').astype(numpy.double)
numpy.save(os.path.join(output_path, 'whitening_mat'),
whitening_mat)
numpy.save(os.path.join(output_path, 'whitening_mat_inv'),
numpy.linalg.inv(whitening_mat))
else:
numpy.save(os.path.join(output_path, 'whitening_mat'),
numpy.eye(N_e))
positions, shanks = generate_mapping(probe)
numpy.save(os.path.join(output_path, 'channel_positions'),
positions.astype(numpy.double))
numpy.save(os.path.join(output_path, 'channel_shanks'),
shanks.astype(numpy.double))
nodes, edges = get_nodes_and_edges(params)
numpy.save(os.path.join(output_path, 'channel_map'),
nodes.astype(numpy.int32))
write_results(output_path, params, extension)
N_tm = write_templates(output_path, params, extension)
template_file = h5py.File(file_out_suff +
'.templates%s.hdf5' % extension,
'r',
libver='earliest')
similarities = template_file.get('maxoverlap')[:]
template_file.close()
norm = N_e * N_t
if export_all:
to_write = numpy.zeros((N_tm + N_e, N_tm + N_e),
dtype=numpy.single)
to_write[:N_tm, :N_tm] = (similarities[:N_tm, :N_tm] /
norm).astype(numpy.single)
else:
to_write = (similarities[:N_tm, :N_tm] / norm).astype(
numpy.single)
numpy.save(os.path.join(output_path, 'similar_templates'),
to_write)
comm.bcast(numpy.array([N_tm], dtype=numpy.int32), root=0)
else:
N_tm = int(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0))
comm.Barrier()
make_pcs = 2
if comm.rank == 0:
if export_pcs == 'prompt':
key = ''
while key not in ['a', 's', 'n']:
print((
Fore.WHITE +
"Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o"
))
key = input('')
else:
key = export_pcs
if key == 'a':
make_pcs = 0
comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
elif key == 's':
make_pcs = 1
comm.bcast(numpy.array([1], dtype=numpy.int32), root=0)
elif key == 'n':
comm.bcast(numpy.array([2], dtype=numpy.int32), root=0)
if os.path.exists(os.path.join(output_path,
'pc_features.npy')):
os.remove(os.path.join(output_path, 'pc_features.npy'))
if os.path.exists(
os.path.join(output_path, 'pc_feature_ind.npy')):
os.remove(os.path.join(output_path, 'pc_feature_ind.npy'))
else:
make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)
make_pcs = make_pcs[0]
comm.Barrier()
if make_pcs < 2:
write_pcs(output_path, params, extension, N_tm, make_pcs)
supported_by_phy = ['raw_binary', 'mcs_raw_binary', 'mda']
file_format = data_file.description
gui_params = {}
if file_format in supported_by_phy:
if not params.getboolean('data', 'overwrite'):
gui_params['dat_path'] = r"%s" % params.get(
'data', 'data_file_no_overwrite')
else:
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True,
has_been_created=False)
gui_params['dat_path'] = "["
for f in data_file.get_file_names():
gui_params['dat_path'] += 'r"%s", ' % f
gui_params['dat_path'] += "]"
else:
gui_params['dat_path'] = 'r"%s"' % params.get(
'data', 'data_file')
else:
gui_params['dat_path'] = 'giverandomname.dat'
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_file.data_dtype
if 'data_offset' in list(data_file.params.keys()):
gui_params['offset'] = data_file.data_offset
gui_params['sample_rate'] = params.rate
gui_params['dir_path'] = output_path
gui_params['hp_filtered'] = True
f = open(os.path.join(output_path, 'params.py'), 'w')
for key, value in list(gui_params.items()):
if key in ['dir_path', 'dtype']:
f.write('%s = r"%s"\n' % (key, value))
else:
f.write("%s = %s\n" % (key, value))
f.close()
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(
['You need to update phy-contrib to the latest git version'],
'error', logger)
sys.exit(1)
if not test_patch_for_similarities(params, extension):
print_and_log(
['You should re-export the data because of a fix in 0.6'], 'error',
logger)
continue_anyway = query_yes_no(
Fore.WHITE + "Continue anyway (results may not be fully correct)?",
default=None)
if not continue_anyway:
sys.exit(1)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
if data_file.params.has_key('data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
file_format = data_file.description
file_out_suff = params.get('data', 'file_out_suff')
if file_format not in supported_by_phy:
print_and_log([
"File format %s is not supported by phy. TraceView disabled" %
file_format
], 'info', logger)
if numpy.iterable(data_file.gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
else:
if data_file.gain != 1:
print_and_log([
"Gain of %g is not supported by phy. Expecting a scaling mismatch"
% data_file.gain
], 'info', logger)
gain = data_file.gain
probe = params.probe
if extension != '':
extension = '-' + extension
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(
['Data should be first exported with the converting method!'],
'error', logger)
else:
print_and_log(["Launching the phy GUI..."], 'info', logger)
gui_params = {}
if file_format in supported_by_phy:
if not params.getboolean('data', 'overwrite'):
gui_params['dat_path'] = params.get('data',
'data_file_no_overwrite')
else:
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True,
has_been_created=False)
gui_params['dat_path'] = ' '.join(
data_file.get_file_names())
else:
gui_params['dat_path'] = params.get('data', 'data_file')
else:
gui_params['dat_path'] = 'giverandomname.dat'
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_dtype
gui_params['offset'] = data_offset
gui_params['sample_rate'] = params.rate
gui_params['dir_path'] = output_path
gui_params['hp_filtered'] = True
f = open(os.path.join(output_path, 'params.py'), 'w')
for key, value in gui_params.items():
if key in ['dir_path', 'dat_path', 'dtype']:
f.write('%s = "%s"\n' % (key, value))
else:
f.write("%s = %s\n" % (key, value))
f.close()
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parallel_hdf5 = h5py.get_config().mpi
user_path = pjoin(os.path.expanduser('~'), 'spyking-circus')
tasks_list = None
if not os.path.exists(user_path):
os.makedirs(user_path)
try:
import cudamat as cmt
cmt.init()
HAVE_CUDA = True
except Exception:
HAVE_CUDA = False
all_steps = [
'whitening', 'clustering', 'fitting', 'gathering', 'extracting',
'filtering', 'converting', 'deconverting', 'benchmarking',
'merging', 'validating', 'thresholding'
]
config_file = os.path.abspath(pkg_resources.resource_filename('circus', 'config.params'))
header = get_colored_header()
header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str(psutil.cpu_count()) + '\n'
# header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str(HAVE_CUDA) + '\n'
header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str(parallel_hdf5) + '\n'
do_upgrade = ''
if not SHARED_MEMORY:
do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]'
header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str(SHARED_MEMORY) + do_upgrade + '\n'
header += '\n'
header += Fore.GREEN + "##################################################################"
header += Fore.RESET
method_help = '''by default, all steps are performed,
but a subset x,y can be done. Steps are:
- filtering
- whitening
- clustering
- fitting
- merging [with or without a GUI for meta merging]
- (extra) converting [export results to phy format]
- (extra) thresholding [to get MUA activity only]
- (extra) deconverting [import results from phy format]
- (extra) gathering [force collection of results]
- (extra) extracting [get templates from spike times]
- (extra) benchmarking [with -o and -t]
- (extra) validating [to compare performance with GT neurons]'''
parser = argparse.ArgumentParser(description=header,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file (or a list of commands if batch mode)')
parser.add_argument('-i', '--info', help='list the file formats supported by SpyKING CIRCUS', action='store_true')
parser.add_argument('-m', '--method',
default='filtering,whitening,clustering,fitting,merging',
help=method_help)
parser.add_argument('-c', '--cpu', type=int, default=max(1, int(psutil.cpu_count()/2)), help='number of CPU')
# parser.add_argument('-g', '--gpu', type=int, default=0, help='number of GPU')
parser.add_argument('-H', '--hostfile', help='hostfile for MPI',
default=pjoin(user_path, 'circus.hosts'))
parser.add_argument('-b', '--batch', help='datafile is a list of commands to launch, in a batch mode',
action='store_true')
parser.add_argument('-p', '--preview', help='GUI to display the first second filtered with thresholds',
action='store_true')
parser.add_argument('-r', '--result', help='GUI to display the results on top of raw data',
action='store_true')
parser.add_argument('-s', '--second', type=int, default=0, help='If preview mode, begining of the preview [in s]')
parser.add_argument('-e', '--extension', help='extension to consider for merging, converting and deconverting',
default='None')
parser.add_argument('-o', '--output', help='output file [for generation of synthetic benchmarks]')
parser.add_argument('-t', '--type', help='benchmark type',
choices=['fitting', 'clustering', 'synchrony'])
if len(argv) == 0:
parser.print_help()
sys.exit(0)
args = parser.parse_args(argv)
steps = args.method.split(',')
for step in steps:
if step not in all_steps:
print_error(['The method "%s" is not recognized' % step])
sys.exit(0)
# To save some typing later
nb_gpu = 0
(nb_cpu, hostfile, batch, preview, result, extension, output, benchmark, info, second) = \
(args.cpu, args.hostfile, args.batch, args.preview, args.result, args.extension, args.output, args.type, args.info, args.second)
filename = os.path.abspath(args.datafile)
real_file = filename
f_next, extens = os.path.splitext(filename)
if info:
if args.datafile.lower() in __supported_data_files__:
filename = 'tmp'
if len(__supported_data_files__[args.datafile.lower()].extension) > 0:
filename += __supported_data_files__[args.datafile.lower()].extension[0]
__supported_data_files__[args.datafile.lower()](filename, {}, is_empty=True)._display_requirements_()
else:
print_and_log([
'',
'To get info on any particular file format, do:',
'>> spyking-circus file_format -i',
''
], 'default')
print_and_log(list_all_file_format())
sys.exit(0)
if extens == '.params':
print_error(['You should launch the code on the data file!'])
sys.exit(0)
file_params = f_next + '.params'
if not os.path.exists(file_params) and not batch:
print(Fore.RED + 'The parameter file %s is not present!' % file_params)
create_params = query_yes_no(Fore.WHITE + "Do you want SpyKING CIRCUS to create a parameter file?")
if create_params:
print(Fore.WHITE + "Creating %s" % file_params)
print(Fore.WHITE + "Fill it properly before launching the code! (see documentation)")
print_info(['Keep in mind that filtering is performed on site, so please',
'be sure to keep a copy of your data elsewhere'])
shutil.copyfile(config_file, file_params)
sys.exit(0)
elif batch:
tasks_list = filename
if not batch:
file_params = f_next + '.params'
if not os.path.exists(file_params):
print_and_log(["%s does not exist" % file_params], 'error')
sys.exit(0)
import ConfigParser as configparser
parser = configparser.ConfigParser()
myfile = open(file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
parser.read(file_params)
for section in CircusParser.__all_sections__:
if parser.has_section(section):
for (key, value) in parser.items(section):
parser.set(section, key, value.split('#')[0].rstrip())
else:
parser.add_section(section)
try:
use_output_dir = parser.get('data', 'output_dir') != ''
except Exception:
use_output_dir = False
if use_output_dir:
path = os.path.abspath(os.path.expanduser(parser.get('data', 'output_dir')))
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out):
os.makedirs(file_out)
else:
file_out = f_next
logfile = file_out + '.log'
if os.path.exists(logfile):
os.remove(logfile)
logger = init_logging(logfile)
params = CircusParser(filename)
data_file = params.get_data_file(source=True, has_been_created=False)
overwrite = params.getboolean('data', 'overwrite')
file_format = params.get('data', 'file_format')
if overwrite:
support_parallel_write = data_file.parallel_write
is_writable = data_file.is_writable
else:
support_parallel_write = __supported_data_files__['raw_binary'].parallel_write
is_writable = __supported_data_files__['raw_binary'].is_writable
if preview:
print_and_log(['Preview mode, showing only seconds [%d-%d] of the recording' % (second, second+1)], 'info', logger)
tmp_path_loc = os.path.join(os.path.abspath(params.get('data', 'file_out')), 'tmp')
if not os.path.exists(tmp_path_loc):
os.makedirs(tmp_path_loc)
filename = os.path.join(tmp_path_loc, 'preview.dat')
f_next, extens = os.path.splitext(filename)
preview_params = f_next + '.params'
shutil.copyfile(file_params, preview_params)
steps = ['filtering', 'whitening']
chunk_size = int(params.rate)
data_file.open()
nb_chunks, _ = data_file.analyze(chunk_size)
if nb_chunks <= (second + 1):
print_and_log(['Recording is too short to display seconds [%d-%d]' % (second, second+1)])
sys.exit(0)
local_chunk = data_file.get_snippet(int(second*params.rate), int(1.2*chunk_size))
description = data_file.get_description()
data_file.close()
new_params = CircusParser(filename, create_folders=False)
new_params.write('data', 'chunk_size', '1')
new_params.write('data', 'file_format', 'raw_binary')
new_params.write('data', 'data_dtype', 'float32')
new_params.write('data', 'data_offset', '0')
new_params.write('data', 'dtype_offset', '0')
new_params.write('data', 'stream_mode', 'None')
new_params.write('data', 'overwrite', 'True')
new_params.write('triggers', 'ignore_times', 'False')
new_params.write('data', 'sampling_rate', str(params.rate))
new_params.write('whitening', 'safety_time', '0')
new_params.write('clustering', 'safety_time', '0')
new_params.write('whitening', 'chunk_size', '1')
new_params.write('data', 'preview_path', params.file_params)
new_params.write('data', 'output_dir', '')
description['data_dtype'] = 'float32'
description['dtype_offset'] = 0
description['data_offset'] = 0
description['gain'] = 1.
new_params = CircusParser(filename)
data_file_out = new_params.get_data_file(is_empty=True, params=description)
support_parallel_write = data_file_out.parallel_write
is_writable = data_file_out.is_writable
data_file_out.allocate(shape=local_chunk.shape, data_dtype=numpy.float32)
data_file_out.open('r+')
data_file_out.set_data(0, local_chunk)
data_file_out.close()
if tasks_list is not None:
with open(tasks_list, 'r') as f:
for line in f:
if len(line) > 0:
subprocess.check_call(['spyking-circus'] + line.replace('\n', '').split(" "))
else:
print_and_log(['Config file: %s' % (f_next + '.params')], 'debug', logger)
print_and_log(['Data file : %s' % filename], 'debug', logger)
print(get_colored_header())
print(Fore.GREEN + "File : " + Fore.CYAN + real_file)
if preview:
print(Fore.GREEN + "Steps : " + Fore.CYAN + "preview mode")
elif result:
print(Fore.GREEN + "Steps : " + Fore.CYAN + "result mode")
else:
print(Fore.GREEN + "Steps : " + Fore.CYAN + ", ".join(steps))
# print Fore.GREEN + "GPU detected : ", Fore.CYAN + str(HAVE_CUDA)
print(Fore.GREEN + "Number of CPU : " + Fore.CYAN + str(nb_cpu) + "/" + str(psutil.cpu_count()))
# if HAVE_CUDA:
# print Fore.GREEN + "Number of GPU : ", Fore.CYAN + str(nb_gpu)
print(Fore.GREEN + "Parallel HDF5 : " + Fore.CYAN + str(parallel_hdf5))
do_upgrade = ''
use_shared_memory = get_shared_memory_flag(params)
if not SHARED_MEMORY:
do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]'
print(Fore.GREEN + "Shared memory : " + Fore.CYAN + str(use_shared_memory) + do_upgrade)
print(Fore.GREEN + "Hostfile : " + Fore.CYAN + hostfile)
print("")
print(Fore.GREEN + "##################################################################")
print("")
print(Fore.RESET)
# Launch the subtasks
subtasks = [('filtering', 'mpirun'),
('whitening', 'mpirun'),
('clustering', 'mpirun'),
('fitting', 'mpirun'),
('extracting', 'mpirun'),
('gathering', 'python'),
('converting', 'mpirun'),
('deconverting', 'mpirun'),
('benchmarking', 'mpirun'),
('merging', 'mpirun'),
('validating', 'mpirun'),
('thresholding', 'mpirun')]
# if HAVE_CUDA and nb_gpu > 0:
# use_gpu = 'True'
# else:
use_gpu = 'False'
time = data_stats(params) / 60.0
if preview:
params = new_params
if nb_cpu < psutil.cpu_count():
if use_gpu != 'True' and not result:
print_and_log(['Using only %d out of %d local CPUs available (-c to change)' % (nb_cpu, psutil.cpu_count())], 'info', logger)
if params.getboolean('detection', 'matched-filter') and not params.getboolean('clustering', 'smart_search'):
print_and_log(['Smart Search should be activated for matched filtering'], 'info', logger)
if time > 30 and not params.getboolean('clustering', 'smart_search'):
print_and_log(['Smart Search should be activated for long recordings'], 'info', logger)
n_edges = get_averaged_n_edges(params)
if n_edges > 100 and not params.getboolean('clustering', 'compress'):
print_and_log(['Template compression is highly recommended based on parameters'], 'info', logger)
if not result:
for subtask, command in subtasks:
if subtask in steps:
if command == 'python':
# Directly call the launcher
try:
circus.launch(subtask, filename, nb_cpu, nb_gpu, use_gpu)
except:
print_and_log(['Step "%s" failed!' % subtask], 'error', logger)
sys.exit(0)
elif command == 'mpirun':
# Use mpirun to make the call
mpi_args = gather_mpi_arguments(hostfile, params)
one_cpu = False
if subtask in ['filtering', 'benchmarking'] and not is_writable:
if not preview and overwrite:
print_and_log(['The file format %s is read only!' % file_format,
'You should set overwite to False, to create a copy of the data.',
'However, note that if you have streams, informations on times',
'will be discarded'], 'info', logger)
sys.exit(0)
if subtask in ['filtering'] and not support_parallel_write and (args.cpu > 1):
print_and_log(['No parallel writes for %s: only 1 node used for %s' %(file_format, subtask)], 'info', logger)
nb_tasks = str(1)
one_cpu = True
else:
if subtask != 'fitting':
nb_tasks = str(args.cpu)
else:
# if use_gpu == 'True':
# nb_tasks = str(args.gpu)
# else:
nb_tasks = str(args.cpu)
if subtask == 'benchmarking':
if (output is None) or (benchmark is None):
print_and_log(["To generate synthetic datasets, you must provide output and type"], 'error', logger)
sys.exit(0)
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, output, benchmark
]
elif subtask in ['merging', 'converting']:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, extension
]
elif subtask in ['deconverting']:
nb_tasks = str(1)
nb_cpu = 1
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask', subtask,
filename, str(nb_cpu), str(nb_gpu), use_gpu,
extension
]
else:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, str(one_cpu)
]
print_and_log(['Launching task %s' % subtask], 'debug', logger)
print_and_log(['Command: %s' % str(mpi_args)], 'debug', logger)
try:
subprocess.check_call(mpi_args)
except subprocess.CalledProcessError as e:
print_and_log(['Step "%s" failed for reason %s!' % (subtask, e)], 'error', logger)
sys.exit(0)
if preview or result:
from circus.shared import gui
import pylab
try:
from PyQt5.QtWidgets import QApplication
except ImportError:
from matplotlib.backends import qt_compat
use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE
if use_pyside:
from PySide.QtGui import QApplication
else:
from PyQt4.QtGui import QApplication
app = QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
if preview:
print_and_log(['Launching the preview GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(new_params)
shutil.rmtree(tmp_path_loc)
elif result:
data_file = params.get_data_file()
print_and_log(['Launching the result GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(params, show_fit=True)
sys.exit(app.exec_())
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to group files within several folders into a single
virtual folder, such that they can be processed together with the
multi-files mode.
If you want to also process .dead or .trig files in order to later
on concatenate artefacts, please use the -d or -t options
'''
parser = argparse.ArgumentParser(description=header,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('folders', help='text file with the list of folders to consider')
parser.add_argument('extension', help='file extension to consider within folders')
parser.add_argument('-o', '--output', help='name of the output folder [default is output]', default='output')
parser.add_argument('-d', '--dead', help='Search for all .dead files', action='store_true')
parser.add_argument('-t', '--trig', help='Search for all .trig files', action='store_true')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
folders_file = os.path.abspath(args.folders)
output = os.path.abspath(args.output)
extension = args.extension
filename, ext = os.path.splitext(os.path.basename(folders_file))
logger = init_logging(filename + '.log')
logger = logging.getLogger(__name__)
if not os.path.exists(folders_file):
print_and_log(['The folder file %s does not exists!' %folders_file], 'error', logger)
sys.exit(0)
try:
folders = []
myfile = open(folders_file, 'r')
lines = myfile.readlines()
myfile.close()
for l in lines:
folders += [os.path.abspath(l.strip())]
except Exception:
print_and_log(['Check the syntax of the folder file'], 'error', logger)
sys.exit(0)
do_folders = True
if os.path.exists(output):
do_folders = query_yes_no(Fore.WHITE + "Folder %s already exists! Do you want to erase everything?" %output, default=None)
if not do_folders:
sys.exit(0)
else:
shutil.rmtree(output)
os.makedirs(output)
for count, folder in enumerate(folders):
files = os.listdir(folder)
for file in files:
_, ext = os.path.splitext(file)
ext = ext.strip('.')
if (ext.lower() == extension.lower()) or (args.dead and ext.lower() == 'dead') or (args.trig and ext.lower()== 'trig'):
original_file = os.path.join(folder, file)
linked_file = os.path.join(output, 'sc_{c}_{f}'.format(c=count, f=os.path.basename(original_file)))
if not os.path.exists(linked_file):
os.symlink(original_file, linked_file)
else:
os.symlink(original_file, linked_file)
def main(params, nb_cpu, nb_gpu, use_gpu, extension):
_ = init_logging(params.logfile)
logger = logging.getLogger('circus.merging')
file_out_suff = params.get('data', 'file_out_suff')
erase_all = params.getboolean('merging', 'erase_all')
extension_in = extension
extension_out = '-merged'
# Erase previous results (if user agrees).
if comm.rank == 0:
existing_file_paths = [
file_path for file_path in [
file_out_suff + ".%s%s.hdf5" % (file_id, extension_out)
for file_id in ['templates', 'clusters', 'result']
] if os.path.isfile(file_path)
]
existing_directory_path = [
directory_path
for directory_path in [file_out_suff + "%s.GUI" % extension_out]
if os.path.isdir(directory_path)
]
if len(existing_file_paths) > 0 or len(existing_directory_path) > 0:
if not erase_all:
erase = query_yes_no(
"Merging already done! Do you want to erase previous merging results?",
default=None)
else:
erase = True
if erase:
for path in existing_file_paths:
os.remove(path)
if comm.rank == 0:
print_and_log(["Removed file %s" % path], 'debug',
logger)
for path in existing_directory_path:
shutil.rmtree(path)
if comm.rank == 0:
print_and_log(["Removed directory %s" % path], 'debug',
logger)
comm.Barrier()
if params.getfloat('merging', 'auto_mode') == 0:
from sys import platform
if not platform == 'win32':
if not ('DISPLAY' in os.environ
and re.search(":\d", os.environ['DISPLAY']) != None):
print_and_log(
['Preview mode can not be used, check DISPLAY variable'],
'error', logger)
sys.exit(0)
if comm.rank == 0:
try:
from PyQt5.QtWidgets import QApplication
except ImportError:
from matplotlib.backends import qt_compat
use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE
if use_pyside:
from PySide.QtGui import QApplication
else:
from PyQt4.QtGui import QApplication
app = QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
else:
app = None
else:
app = None
if comm.rank == 0:
print_and_log(['Launching the merging GUI...'], 'debug', logger)
_ = gui.MergeWindow(params, app, extension_in, extension_out)
sys.exit(app.exec_())
def main(params, nb_cpu, nb_gpu, use_gpu, extension):
assert comm.rank == 0
input_extension = extension
logger = init_logging(params.logfile)
logger = logging.getLogger('circus.deconverting')
# Retrieve parameters.
input_path = params.get('data', 'file_out_suff') + input_extension + '.GUI'
output_path = params.get('data', 'file_out_suff')
output_extension = '-deconverted'
clusters_path = output_path + '.clusters{}.hdf5'.format(output_extension)
templates_path = output_path + '.templates{}.hdf5'.format(output_extension)
result_path = output_path + '.result{}.hdf5'.format(output_extension)
# Check if input path exists.
if not os.path.isdir(input_path):
print_and_log([
"Can't find directory: {}".format(input_path),
"You must first export your results into the phy format (use the converting method)."
], 'error', logger)
sys.exit(0)
# Check if results are already present.
if os.path.isfile(clusters_path) \
and os.path.isfile(templates_path) \
and os.path.isfile(result_path):
print_and_log([
"Phy results already imported.",
"Delete the following files to run the deconversion again:",
" - {}".format(result_path),
" - {}".format(templates_path),
" - {}".format(clusters_path),
], 'error', logger)
if query_yes_no("Do you want to delete these files?"):
os.remove(result_path)
os.remove(templates_path)
os.remove(clusters_path)
else:
sys.exit(0)
# Check if results are partially present.
if os.path.isfile(clusters_path) \
or os.path.isfile(templates_path) \
or os.path.isfile(result_path):
print_and_log([
"Phy results partially imported.",
"Delete the following files to be able to run the deconversion:",
] + [
" - {}".format(path)
for path in [result_path, templates_path, clusters_path]
if os.path.isfile(path)
], 'error', logger)
if query_yes_no("Do you want to delete these files?"):
for path in [result_path, templates_path, clusters_path]:
if os.path.isfile(path):
os.remove(path)
else:
sys.exit(0)
# Log introduction message.
message = "Importing data from the phy GUI with {} CPUs...".format(nb_cpu)
print_and_log([message], 'info', logger)
# Read phy results.
phy_results = load_phy_results(input_path)
spike_templates = phy_results['spike_templates']
spike_clusters = phy_results['spike_clusters']
cluster_group = phy_results['cluster_group']
templates = phy_results['templates']
# print_and_log(["{}".format(phy_results)], 'debug', logger)
# Read spyking-circus results.
templates_input_path = output_path + ".templates{}.hdf5".format(
input_extension)
templates_input_file = h5py.File(templates_input_path,
mode='r',
libver='earliest')
overlaps = templates_input_file.get('maxoverlap')[:]
try:
lag = templates_input_file.get('maxlag')[:]
except TypeError: # i.e. 'maxlag' not in HDF5 file.
lag = np.zeros(overlaps.shape, dtype=np.int32)
shape = templates_input_file.get('temp_shape')[:]
# Set correct lag option.
correct_lag = True
# Determine association map between spike templates and spike clusters.
templates_to_clusters = {}
for spike_template, spike_cluster in zip(spike_templates, spike_clusters):
templates_to_clusters[spike_template] = spike_cluster
# Determine association map between spike cluster and default spike template.
clusters_to_templates = {}
for spike_cluster in cluster_group:
clusters_to_templates[spike_cluster] = None
# # TODO remove templates without spikes.
# print_and_log([ # TODO remove.
# "Removing templates without spikes..."
# ], 'info', logger)
# electrodes = io.load_data(params, 'electrodes', extension=input_extension) # TODO remove duplicate.
# clusters = io.load_data(params, 'clusters', extension=input_extension) # TODO remove duplicate
# for spike_template, _ in templates_to_clusters.items():
# # Retrieve the prefered electrode.
# elec_ic = electrodes[spike_template]
# # Retrieve template index among templates with same prefered electrodeself.
# first_index = np.where(electrodes == elec_ic)[0][0]
# nic = spike_template - first_index
# # Retrieve the cluster label.
# label = 'clusters_{}'.format(elec_ic)
# # Select the points labelled by the clustering.
# mask = clusters[label] > -1
# # Retrieve the labels used by the clustering.
# tmp = np.unique(clusters[label][mask])
# # Retrieve the number of points labelled for both templates.
# cluster_label = tmp[nic]
# elements = np.where(clusters[label] == cluster_label)[0]
# # ...
# if len(elements) == 0:
# print_and_log([
# "template {} has no spike".format(spike_template)
# ], 'info', logger)
# raise NotImplementedError
to_merge = []
to_remove = []
# Do all the merges.
old_results = io.load_data(params, 'results', extension=input_extension)
electrodes = io.load_data(params, 'electrodes', extension=input_extension)
clusters = io.load_data(params, 'clusters', extension=input_extension)
for spike_template, spike_cluster in templates_to_clusters.items():
spike_group = cluster_group[spike_cluster]
if spike_group in ['good', 'unsorted']:
if clusters_to_templates[spike_cluster] is None:
clusters_to_templates[spike_cluster] = spike_template
else:
# Retrieve pair of templates to merge.
default_spike_template = clusters_to_templates[spike_cluster]
one_merge = [default_spike_template, spike_template]
# Retrieve the prefered electrode for both template.
elec_ic1 = electrodes[one_merge[0]]
elec_ic2 = electrodes[one_merge[1]]
# Retrieve template index among templates with same prefered electrode for both templates.
first_index1 = np.where(electrodes == elec_ic1)[0][0]
first_index2 = np.where(electrodes == elec_ic2)[0][0]
nic1 = one_merge[0] - first_index1
nic2 = one_merge[1] - first_index2
# Retrieve the cluster label for both templates.
label1 = 'clusters_{}'.format(elec_ic1)
label2 = 'clusters_{}'.format(elec_ic2)
# Select the points labelled by the clustering for both templates.
mask1 = clusters[label1] > -1
mask2 = clusters[label2] > -1
# Retrieve the labels used by the clustering for both templates.
tmp1 = np.unique(clusters[label1][mask1])
tmp2 = np.unique(clusters[label2][mask2])
# Retrieve the number of points labelled for both templates.
cluster_label1 = tmp1[nic1]
cluster_label2 = tmp2[nic2]
elements1 = np.where(clusters[label1] == cluster_label1)[0]
elements2 = np.where(clusters[label2] == cluster_label2)[0]
# Determine index to keep and index to delete.
if len(elements1) > len(elements2):
to_delete = one_merge[1]
to_keep = one_merge[0]
elec = elec_ic2
elements = elements2
else:
to_delete = one_merge[0]
to_keep = one_merge[1]
elec = elec_ic1
elements = elements1
# print_and_log([
# "one_merge: {}".format(one_merge),
# "elec_ic1, elec_ic2: {}, {}".format(elec_ic1, elec_ic2),
# "nic1, nic2: {}, {}".format(nic1, nic2),
# "tmp1, tmp2: {}, {}".format(tmp1, tmp2),
# "cluster_label1, cluster_label2: {}, {}".format(cluster_label1, cluster_label2),
# "to_keep, to_delete: {}, {}".format(to_keep, to_delete)
# ] , 'debug', logger
# )
# Merge templates (if necessary).
if to_keep != to_delete:
key1 = 'temp_{}'.format(to_keep)
key2 = 'temp_{}'.format(to_delete)
amplitudes1 = old_results['amplitudes'][key1]
amplitudes2 = old_results['amplitudes'][key2]
old_results['amplitudes'][key1] = np.vstack(
(amplitudes1, amplitudes2))
spiketimes1 = old_results['spiketimes'][key1]
spiketimes2 = old_results['spiketimes'][key2]
if correct_lag:
spiketimes2 = spiketimes2.astype(np.int64)
spiketimes2 += lag[to_keep, to_delete]
spiketimes2 = spiketimes2.astype(np.uint32)
old_results['spiketimes'][key1] = np.concatenate(
(spiketimes1, spiketimes2))
indices = np.argsort(old_results['spiketimes'][key1])
old_results['amplitudes'][key1] = old_results[
'amplitudes'][key1][indices]
old_results['spiketimes'][key1] = old_results[
'spiketimes'][key1][indices]
old_results['amplitudes'].pop(key2)
old_results['spiketimes'].pop(key2)
# Update internal variables.
clusters_to_templates[spike_cluster] = to_keep
to_merge.append((to_keep, to_delete))
elif spike_group in ['mua', 'noise']:
to_remove.append(spike_template)
else:
message = "Unexpected group value: {}".format(spike_group)
raise ValueError(message)
# Remove unmentioned templates (e.g. without any fitted spike).
old_templates = load_data(params, 'templates', extension=input_extension)
initial_nb_templates = old_templates.shape[1] // 2
all_spike_templates = set(range(0, initial_nb_templates))
mentioned_spike_templates = set(templates_to_clusters.keys())
unmentioned_spike_templates = list(all_spike_templates -
mentioned_spike_templates)
# print_and_log(["unmentioned templates: {}".format(unmentioned_spike_templates)], 'info', logger)
to_remove.extend(unmentioned_spike_templates)
if to_merge == []:
to_merge = np.zeros((0, 2), dtype=np.int)
else:
to_merge = np.array(to_merge)
to_merge = to_merge[np.lexsort((to_merge[:, 1], to_merge[:, 0])), :]
to_remove.sort()
# Log some information.
nb_merges = to_merge.shape[0]
nb_removals = len(to_remove)
final_nb_templates = initial_nb_templates - nb_merges - nb_removals
print_and_log([
"Manual sorting with the Python GUI (i.e. phy):",
" initial number of templates: {}".format(initial_nb_templates),
" number of merges: {}".format(nb_merges),
" number of removals: {}".format(nb_removals),
" final number of templates: {}".format(final_nb_templates),
], 'info', logger)
# Slice templates.
to_keep = slice_templates(params,
to_merge=to_merge,
to_remove=to_remove,
extension=output_extension,
input_extension=extension)
# print_and_log([
# "to_merge (passed to slice_templates: {}".format(to_merge),
# "to_remove (passed to slice_templates: {}".format(to_remove),
# "to_keep (returned be slice_templates): {}".format(to_keep),
# ], 'info', logger)
# Slice clusters.
light = True
dataname = 'clusters-light' if light else 'clusters'
clusters = io.load_data(params, dataname, extension=extension)
slice_clusters(params,
clusters,
to_merge=to_merge,
to_remove=to_remove,
extension=output_extension,
input_extension=extension,
light=light,
method='new')
# Finalize result.
# nb_templates = templates.shape[0]
# template_indices = np.arange(0, nb_templates)
# to_delete = list(to_remove)
# for one_merge in to_merge:
# to_delete.append(one_merge[1])
# to_keep = set(np.unique(template_indices)) - set(to_delete)
# to_keep = np.array(list(to_keep))
# to_keep = np.sort(to_keep)
# # Would be correct if we could sort 'to_keep' in 'slice_templates'.
# print_and_log([
# "to_keep: {}".format(to_keep)
# ], 'idebug', logger)
new_results = {
'spiketimes': {},
'amplitudes': {},
}
for k, template_index in enumerate(to_keep):
old_key = 'temp_{}'.format(template_index)
new_key = 'temp_{}'.format(k)
new_results['spiketimes'][new_key] = old_results['spiketimes'].pop(
old_key)
new_results['amplitudes'][new_key] = old_results['amplitudes'].pop(
old_key)
# Check if the number of spikes is not equal to 0.
nb_spikes = len(new_results['spiketimes'][new_key])
if nb_spikes == 0:
print_and_log(
["{} - template {} has no spikes".format(k, template_index)],
'error', logger)
keys = ['spiketimes', 'amplitudes']
# TODO add support for [fitting] collect_all=True (not supported).
# Save new result to output file.
result_output_path = output_path + ".result{}.hdf5".format(
output_extension)
result_output_file = h5py.File(result_output_path,
mode='w',
libver='earliest')
for key in keys:
result_output_file.create_group(key)
for temp in new_results[key].keys():
tmp_path = "{}/{}".format(key, temp)
result_output_file.create_dataset(tmp_path,
data=new_results[key][temp])
result_output_file.close()
# Add additional information to templates file.
templates_output_path = output_path + ".templates{}.hdf5".format(
output_extension)
templates_output_file = h5py.File(templates_output_path,
mode='r+',
libver='earliest')
new_shape = (len(to_keep), len(to_keep))
version = templates_output_file.create_dataset(
'version',
data=np.array(circus.__version__.split('.'), dtype=np.int32))
maxoverlaps = templates_output_file.create_dataset('maxoverlap',
shape=new_shape,
dtype=np.float32)
maxlag = templates_output_file.create_dataset('maxlag',
shape=new_shape,
dtype=np.int32)
for k, index in enumerate(to_keep):
maxoverlaps[k, :] = overlaps[index, to_keep]
maxlag[k, :] = lag[index, to_keep]
templates_output_file.close()
# Log conclusion message.
message = "Data from the phy GUI imported."
print_and_log([message], 'info', logger)
return
