def main(params, nb_cpu, nb_gpu, use_gpu, extension): logger = init_logging(params.logfile) logger = logging.getLogger('circus.merging') file_out_suff = params.get('data', 'file_out_suff') extension_in = extension extension_out = '-merged' if comm.rank == 0: if (extension != '') and (os.path.exists(file_out_suff + '.result%s.hdf5' % extension_out)): erase = query_yes_no( "Export already made! Do you want to erase everything?", default=None) if erase: purge(file_out_suff, extension) extension_in = '' comm.Barrier() if comm.rank == 0: app = QApplication([]) try: pylab.style.use('ggplot') except Exception: pass else: app = None if comm.rank == 0: print_and_log(['Launching the merging GUI...'], 'debug', logger) mygui = gui.MergeWindow(params, app, extension_in, extension_out) sys.exit(app.exec_())
def main(params, nb_cpu, nb_gpu, use_gpu, extension): _ = init_logging(params.logfile) logger = logging.getLogger('circus.merging') file_out_suff = params.get('data', 'file_out_suff') erase_all = params.getboolean('merging', 'erase_all') extension_in = extension extension_out = '-merged' # Erase previous results (if user agrees). if comm.rank == 0: existing_file_paths = [ file_path for file_path in [ file_out_suff + ".%s%s.hdf5" % (file_id, extension_out) for file_id in ['templates', 'clusters', 'result'] ] if os.path.isfile(file_path) ] existing_directory_path = [ directory_path for directory_path in [file_out_suff + "%s.GUI" % extension_out] if os.path.isdir(directory_path) ] if len(existing_file_paths) > 0 or len(existing_directory_path) > 0: if not erase_all: erase = query_yes_no( "Merging already done! Do you want to erase previous merging results?", default=None) else: erase = True if erase: for path in existing_file_paths: os.remove(path) if comm.rank == 0: print_and_log(["Removed file %s" % path], 'debug', logger) for path in existing_directory_path: shutil.rmtree(path) if comm.rank == 0: print_and_log(["Removed directory %s" % path], 'debug', logger) comm.Barrier() if comm.rank == 0 and params.getfloat('merging', 'auto_mode') == 0: app = QApplication([]) try: pylab.style.use('ggplot') except Exception: pass else: app = None if comm.rank == 0: print_and_log(['Launching the merging GUI...'], 'debug', logger) _ = gui.MergeWindow(params, app, extension_in, extension_out) sys.exit(app.exec_())
def main(argv=None): if argv is None: argv = sys.argv[1:] header = get_colored_header() header += '''Utility to launch the phy GUI and visualize the results. [data must be first converted with the converting mode] ''' parser = argparse.ArgumentParser( description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file') parser.add_argument('-e', '--extension', help='extension to consider for visualization', default='') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) filename = os.path.abspath(args.datafile) extension = args.extension params = CircusParser(filename) if os.path.exists(params.logfile): os.remove(params.logfile) logger = init_logging(params.logfile) logger = logging.getLogger(__name__) if extension != '': extension = '-' + extension try: import traitlets except ImportError: print_and_log( ['The package traitlets required by phy is not installed'], 'error', logger) sys.exit(1) try: import click except ImportError: print_and_log(['The package click required by phy is not installed'], 'error', logger) sys.exit(1) try: import joblib except ImportError: print_and_log(['The package joblib required by phy is not installed'], 'error', logger) sys.exit(1) if HAVE_PHYCONTRIB: mytest = StrictVersion( phycontrib.__version__) >= StrictVersion("1.0.12") if not mytest: print_and_log( ['You need to update phy-contrib to the latest git version'], 'error', logger) sys.exit(1) print_and_log([ 'phy-contrib is deprecated, you should upgrade to phy 2.0 and phylib' ], 'info', logger) if HAVE_PHYLIB: try: import colorcet except ImportError: print_and_log( ['The package colorcet required by phy is not installed'], 'error', logger) sys.exit(1) try: import qtconsole except ImportError: print_and_log( ['The package qtconsole required by phy is not installed'], 'error', logger) sys.exit(1) if not test_patch_for_similarities(params, extension): print_and_log( ['You should re-export the data because of a fix in 0.6'], 'error', logger) continue_anyway = query_yes_no( Fore.WHITE + "Continue anyway (results may not be fully correct)?", default=None) if not continue_anyway: sys.exit(1) data_file = params.get_data_file() data_dtype = data_file.data_dtype if data_file.params.has_key('data_offset'): data_offset = data_file.data_offset else: data_offset = 0 file_format = data_file.description file_out_suff = params.get('data', 'file_out_suff') if file_format not in supported_by_phy: print_and_log([ "File format %s is not supported by phy. TraceView disabled" % file_format ], 'info', logger) if numpy.iterable(data_file.gain): print_and_log( ['Multiple gains are not supported, using a default value of 1'], 'info', logger) gain = 1 else: if data_file.gain != 1: print_and_log([ "Gain of %g is not supported by phy. Expecting a scaling mismatch" % data_file.gain ], 'info', logger) gain = data_file.gain probe = params.probe output_path = params.get('data', 'file_out_suff') + extension + '.GUI' if not os.path.exists(output_path): print_and_log( ['Data should be first exported with the converting method!'], 'error', logger) else: print_and_log(["Launching the phy GUI..."], 'info', logger) gui_params = {} if file_format in supported_by_phy: if not params.getboolean('data', 'overwrite'): gui_params['dat_path'] = r"%s" % params.get( 'data', 'data_file_no_overwrite') else: if params.get('data', 'stream_mode') == 'multi-files': data_file = params.get_data_file(source=True, has_been_created=False) gui_params['dat_path'] = [ r"%s" % f for f in data_file.get_file_names() ] else: gui_params['dat_path'] = r"%s" % params.get( 'data', 'data_file') else: gui_params['dat_path'] = 'giverandomname.dat' gui_params['n_channels_dat'] = params.nb_channels gui_params['n_features_per_channel'] = 5 gui_params['dtype'] = data_dtype gui_params['offset'] = data_offset gui_params['sample_rate'] = params.rate gui_params['dir_path'] = output_path gui_params['hp_filtered'] = True os.chdir(output_path) create_app() controller = TemplateController(**gui_params) gui = controller.create_gui() gui.show() run_app() gui.close() del gui
def main(params, nb_cpu, nb_gpu, use_gpu, extension): logger = init_logging(params.logfile) logger = logging.getLogger('circus.converting') data_file = params.data_file file_out_suff = params.get('data', 'file_out_suff') probe = params.probe output_path = params.get('data', 'file_out_suff') + extension + '.GUI' N_e = params.getint('data', 'N_e') N_t = params.getint('detection', 'N_t') erase_all = params.getboolean('converting', 'erase_all') export_pcs = params.get('converting', 'export_pcs') export_all = params.getboolean('converting', 'export_all') sparse_export = params.getboolean('converting', 'sparse_export') if export_all and not params.getboolean('fitting', 'collect_all'): if comm.rank == 0: print_and_log(['Export unfitted spikes only if [fitting] collect_all is True'], 'error', logger) sys.exit(0) def generate_mapping(probe): p = {} positions = [] nodes = [] for key in probe['channel_groups'].keys(): p.update(probe['channel_groups'][key]['geometry']) nodes += probe['channel_groups'][key]['channels'] positions += [p[channel] for channel in probe['channel_groups'][key]['channels']] idx = numpy.argsort(nodes) positions = numpy.array(positions)[idx] return positions def get_max_loc_channel(params): nodes, edges = get_nodes_and_edges(params) max_loc_channel = 0 for key in edges.keys(): if len(edges[key]) > max_loc_channel: max_loc_channel = len(edges[key]) return max_loc_channel def write_results(path, params, extension): result = io.get_results(params, extension) spikes = numpy.zeros(0, dtype=numpy.uint64) clusters = numpy.zeros(0, dtype=numpy.uint32) amplitudes = numpy.zeros(0, dtype=numpy.double) N_tm = len(result['spiketimes']) for key in result['spiketimes'].keys(): temp_id = int(key.split('_')[-1]) data = result['spiketimes'].pop(key).astype(numpy.uint64) spikes = numpy.concatenate((spikes, data)) data = result['amplitudes'].pop(key).astype(numpy.double) amplitudes = numpy.concatenate((amplitudes, data[:, 0])) clusters = numpy.concatenate((clusters, temp_id*numpy.ones(len(data), dtype=numpy.uint32))) if export_all: print_and_log(["Last %d templates are unfitted spikes on all electrodes" %N_e], 'info', logger) garbage = io.load_data(params, 'garbage', extension) for key in garbage['gspikes'].keys(): elec_id = int(key.split('_')[-1]) data = garbage['gspikes'].pop(key).astype(numpy.uint64) spikes = numpy.concatenate((spikes, data)) amplitudes = numpy.concatenate((amplitudes, numpy.zeros(len(data)))) clusters = numpy.concatenate((clusters, (elec_id + N_tm)*numpy.ones(len(data), dtype=numpy.uint32))) idx = numpy.argsort(spikes) numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx]) numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx]) numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx]) return def write_templates(path, params, extension): max_loc_channel = get_max_loc_channel(params) templates = io.load_data(params, 'templates', extension) N_tm = templates.shape[1]//2 if sparse_export: n_channels_max = 0 for t in xrange(N_tm): data = numpy.sum(numpy.sum(templates[:, t].toarray().reshape(N_e, N_t), 1) != 0) if data > n_channels_max: n_channels_max = data else: n_channels_max = N_e if export_all: to_write_sparse = numpy.zeros((N_tm + N_e, N_t, n_channels_max), dtype=numpy.float32) mapping_sparse = -1 * numpy.ones((N_tm + N_e, n_channels_max), dtype=numpy.int32) else: to_write_sparse = numpy.zeros((N_tm, N_t, n_channels_max), dtype=numpy.float32) mapping_sparse = -1 * numpy.ones((N_tm, n_channels_max), dtype=numpy.int32) for t in xrange(N_tm): tmp = templates[:, t].toarray().reshape(N_e, N_t).T x, y = tmp.nonzero() nb_loc = len(numpy.unique(y)) if sparse_export: all_positions = numpy.zeros(y.max()+1, dtype=numpy.int32) all_positions[numpy.unique(y)] = numpy.arange(nb_loc, dtype=numpy.int32) pos = all_positions[y] to_write_sparse[t, x, pos] = tmp[x, y] mapping_sparse[t, numpy.arange(nb_loc)] = numpy.unique(y) else: pos = y to_write_sparse[t, x, pos] = tmp[x, y] if export_all: for t in xrange(N_tm, N_tm + N_e): mapping_sparse[t, 0] = t - N_tm numpy.save(os.path.join(output_path, 'templates'), to_write_sparse) if sparse_export: numpy.save(os.path.join(output_path, 'template_ind'), mapping_sparse) return N_tm def write_pcs(path, params, extension, N_tm, mode=0): spikes = numpy.load(os.path.join(output_path, 'spike_times.npy')) labels = numpy.load(os.path.join(output_path, 'spike_templates.npy')) max_loc_channel = get_max_loc_channel(params) nb_features = params.getint('whitening', 'output_dim') sign_peaks = params.get('detection', 'peaks') nodes, edges = get_nodes_and_edges(params) N_total = params.getint('data', 'N_total') if export_all: nb_templates = N_tm + N_e else: nb_templates = N_tm pc_features_ind = numpy.zeros((nb_templates, max_loc_channel), dtype=numpy.int32) clusters = io.load_data(params, 'clusters', extension) best_elec = clusters['electrodes'] if export_all: best_elec = numpy.concatenate((best_elec, numpy.arange(N_e))) inv_nodes = numpy.zeros(N_total, dtype=numpy.int32) inv_nodes[nodes] = numpy.argsort(nodes) for count, elec in enumerate(best_elec): nb_loc = len(edges[nodes[elec]]) pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[nodes[elec]]] if sign_peaks in ['negative', 'both']: basis_proj, basis_rec = io.load_data(params, 'basis') elif sign_peaks in ['positive']: basis_proj, basis_rec = io.load_data(params, 'basis-pos') to_process = numpy.arange(comm.rank, nb_templates, comm.size) all_offsets = numpy.zeros(nb_templates, dtype=numpy.int32) for target in xrange(nb_templates): if mode == 0: all_offsets[target] = len(numpy.where(labels == target)[0]) elif mode == 1: all_offsets[target] = min(500, len(numpy.where(labels == target)[0])) all_paddings = numpy.concatenate(([0] , numpy.cumsum(all_offsets))) total_pcs = numpy.sum(all_offsets) pc_file = os.path.join(output_path, 'pc_features.npy') pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy') from numpy.lib.format import open_memmap if comm.rank == 0: pc_features = open_memmap(pc_file, shape=(total_pcs, nb_features, max_loc_channel), dtype=numpy.float32, mode='w+') if mode == 1: pc_ids = open_memmap(pc_file_ids, shape=(total_pcs, ), dtype=numpy.int32, mode='w+') comm.Barrier() pc_features = open_memmap(pc_file, mode='r+') if mode == 1: pc_ids = open_memmap(pc_file_ids, mode='r+') to_explore = xrange(comm.rank, nb_templates, comm.size) if comm.rank == 0: to_explore = get_tqdm_progressbar(to_explore) all_idx = numpy.zeros(0, dtype=numpy.int32) for gcount, target in enumerate(to_explore): count = all_paddings[target] if mode == 1: idx = numpy.random.permutation(numpy.where(labels == target)[0])[:500] pc_ids[count:count+len(idx)] = idx elif mode == 0: idx = numpy.where(labels == target)[0] elec = best_elec[target] indices = inv_nodes[edges[nodes[elec]]] labels_i = target*numpy.ones(len(idx)) times_i = numpy.take(spikes, idx).astype(numpy.int64) sub_data = io.get_stas(params, times_i, labels_i, elec, neighs=indices, nodes=nodes, auto_align=False) pcs = numpy.dot(sub_data, basis_proj) pcs = numpy.swapaxes(pcs, 1,2) if mode == 0: pc_features[idx, :, :len(indices)] = pcs elif mode == 1: pc_features[count:count+len(idx), :, :len(indices)] = pcs comm.Barrier() if comm.rank == 0: numpy.save(os.path.join(output_path, 'pc_feature_ind'), pc_features_ind.astype(numpy.uint32)) #n_templates, n_loc_chan do_export = True if comm.rank == 0: if os.path.exists(output_path): if not erase_all: do_export = query_yes_no(Fore.WHITE + "Export already made! Do you want to erase everything?", default=None) if do_export: if os.path.exists(os.path.abspath('.phy')): shutil.rmtree(os.path.abspath('.phy')) shutil.rmtree(output_path) if do_export == True: comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) elif do_export == False: comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) else: do_export = bool(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)) comm.Barrier() if do_export: if comm.rank == 0: os.makedirs(output_path) print_and_log(["Exporting data for the phy GUI with %d CPUs..." %nb_cpu], 'info', logger) if params.getboolean('whitening', 'spatial'): whitening_mat = io.load_data(params, 'spatial_whitening').astype(numpy.double) numpy.save(os.path.join(output_path, 'whitening_mat'), whitening_mat) numpy.save(os.path.join(output_path, 'whitening_mat_inv'), numpy.linalg.inv(whitening_mat)) else: numpy.save(os.path.join(output_path, 'whitening_mat'), numpy.eye(N_e)) numpy.save(os.path.join(output_path, 'channel_positions'), generate_mapping(probe).astype(numpy.double)) nodes, edges = get_nodes_and_edges(params) numpy.save(os.path.join(output_path, 'channel_map'), nodes.astype(numpy.int32)) write_results(output_path, params, extension) N_tm = write_templates(output_path, params, extension) apply_patch_for_similarities(params, extension) template_file = h5py.File(file_out_suff + '.templates%s.hdf5' %extension, 'r', libver='earliest') similarities = template_file.get('maxoverlap')[:] template_file.close() norm = N_e*N_t if export_all: to_write = numpy.zeros((N_tm + N_e, N_tm + N_e), dtype=numpy.single) to_write[:N_tm, :N_tm] = (similarities[:N_tm, :N_tm]/norm).astype(numpy.single) else: to_write = (similarities[:N_tm, :N_tm]/norm).astype(numpy.single) numpy.save(os.path.join(output_path, 'similar_templates'), to_write) comm.bcast(numpy.array([N_tm], dtype=numpy.int32), root=0) else: N_tm = int(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)) comm.Barrier() make_pcs = 2 if comm.rank == 0: if export_pcs == 'prompt': key = '' while key not in ['a', 's', 'n']: print(Fore.WHITE + "Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o") key = raw_input('') else: key = export_pcs if key == 'a': make_pcs = 0 comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) elif key == 's': make_pcs = 1 comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) elif key == 'n': comm.bcast(numpy.array([2], dtype=numpy.int32), root=0) if os.path.exists(os.path.join(output_path, 'pc_features.npy')): os.remove(os.path.join(output_path, 'pc_features.npy')) if os.path.exists(os.path.join(output_path, 'pc_feature_ind.npy')): os.remove(os.path.join(output_path, 'pc_feature_ind.npy')) else: make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) make_pcs = make_pcs[0] comm.Barrier() if make_pcs < 2: write_pcs(output_path, params, extension, N_tm, make_pcs)
def main(params, nb_cpu, nb_gpu, use_gpu, extension): _ = init_logging(params.logfile) logger = logging.getLogger('circus.converting') data_file = params.data_file file_out_suff = params.get('data', 'file_out_suff') probe = params.probe output_path = params.get('data', 'file_out_suff') + extension + '.GUI' N_e = params.getint('data', 'N_e') prelabelling = params.getboolean('converting', 'prelabelling') N_t = params.getint('detection', 'N_t') erase_all = params.getboolean('converting', 'erase_all') export_pcs = params.get('converting', 'export_pcs') export_all = params.getboolean('converting', 'export_all') sparse_export = params.getboolean('converting', 'sparse_export') rpv_threshold = params.getfloat('converting', 'rpv_threshold') if export_all and not params.getboolean('fitting', 'collect_all'): if comm.rank == 0: print_and_log([ 'Export unfitted spikes only if [fitting] collect_all is True' ], 'error', logger) sys.exit(0) def generate_mapping(probe): p = {} positions = [] nodes = [] shanks = [] for key in list(probe['channel_groups'].keys()): p.update(probe['channel_groups'][key]['geometry']) nodes += probe['channel_groups'][key]['channels'] positions += [ p[channel] for channel in probe['channel_groups'][key]['channels'] ] shanks += [key] * len(probe['channel_groups'][key]['channels']) positions = numpy.array(positions) shanks = numpy.array(shanks) return positions, shanks def get_max_loc_channel(params, extension): if test_if_support(params, extension): supports = io.load_data(params, 'supports', extension) max_loc_channel = numpy.sum(supports, 1).max() else: nodes, edges = get_nodes_and_edges(params) max_loc_channel = 0 for key in list(edges.keys()): if len(edges[key]) > max_loc_channel: max_loc_channel = len(edges[key]) return max_loc_channel def write_results(path, params, extension): result = io.get_results(params, extension) spikes = [numpy.zeros(0, dtype=numpy.uint64)] clusters = [numpy.zeros(0, dtype=numpy.uint32)] amplitudes = [numpy.zeros(0, dtype=numpy.double)] N_tm = len(result['spiketimes']) has_purity = test_if_purity(params, extension) rpvs = [] if prelabelling: labels = [] norms = io.load_data(params, 'norm-templates', extension) norms = norms[:len(norms) // 2] if has_purity: purity = io.load_data(params, 'purity', extension) for key in list(result['spiketimes'].keys()): temp_id = int(key.split('_')[-1]) myspikes = result['spiketimes'].pop(key).astype(numpy.uint64) spikes.append(myspikes) myamplitudes = result['amplitudes'].pop(key).astype(numpy.double) amplitudes.append(myamplitudes[:, 0]) clusters.append(temp_id * numpy.ones(len(myamplitudes), dtype=numpy.uint32)) rpv = get_rpv(myspikes, params.data_file.sampling_rate) rpvs += [[temp_id, rpv]] if prelabelling: if has_purity: if rpv <= rpv_threshold: if purity[temp_id] > 0.75: labels += [[temp_id, 'good']] else: if purity[temp_id] > 0.75: labels += [[temp_id, 'mua']] else: labels += [[temp_id, 'noise']] else: median_amp = numpy.median(myamplitudes[:, 0]) std_amp = numpy.std(myamplitudes[:, 0]) if rpv <= rpv_threshold and numpy.abs(median_amp - 1) < 0.25: labels += [[temp_id, 'good']] else: if median_amp < 0.5: labels += [[temp_id, 'mua']] elif norms[temp_id] < 0.1: labels += [[temp_id, 'noise']] if export_all: print_and_log([ "Last %d templates are unfitted spikes on all electrodes" % N_e ], 'info', logger) garbage = io.load_data(params, 'garbage', extension) for key in list(garbage['gspikes'].keys()): elec_id = int(key.split('_')[-1]) data = garbage['gspikes'].pop(key).astype(numpy.uint64) spikes.append(data) amplitudes.append(numpy.ones(len(data))) clusters.append((elec_id + N_tm) * numpy.ones(len(data), dtype=numpy.uint32)) if prelabelling: f = open(os.path.join(output_path, 'cluster_group.tsv'), 'w') f.write('cluster_id\tgroup\n') for l in labels: f.write('%s\t%s\n' % (l[0], l[1])) f.close() # f = open(os.path.join(output_path, 'cluster_rpv.tsv'), 'w') # f.write('cluster_id\trpv\n') # for l in rpvs: # f.write('%s\t%s\n' % (l[0], l[1])) # f.close() spikes = numpy.concatenate(spikes).astype(numpy.uint64) amplitudes = numpy.concatenate(amplitudes).astype(numpy.double) clusters = numpy.concatenate(clusters).astype(numpy.uint32) idx = numpy.argsort(spikes) numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx]) numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx]) numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx]) return def write_templates(path, params, extension): max_loc_channel = get_max_loc_channel(params, extension) templates = io.load_data(params, 'templates', extension) N_tm = templates.shape[1] // 2 nodes, edges = get_nodes_and_edges(params) if sparse_export: n_channels_max = 0 for t in range(N_tm): data = numpy.sum( numpy.sum(templates[:, t].toarray().reshape(N_e, N_t), 1) != 0) if data > n_channels_max: n_channels_max = data else: n_channels_max = N_e if export_all: to_write_sparse = numpy.zeros((N_tm + N_e, N_t, n_channels_max), dtype=numpy.float32) mapping_sparse = -1 * numpy.ones( (N_tm + N_e, n_channels_max), dtype=numpy.int32) else: to_write_sparse = numpy.zeros((N_tm, N_t, n_channels_max), dtype=numpy.float32) mapping_sparse = -1 * numpy.ones( (N_tm, n_channels_max), dtype=numpy.int32) has_purity = test_if_purity(params, extension) if has_purity: purity = io.load_data(params, 'purity', extension) f = open(os.path.join(output_path, 'cluster_purity.tsv'), 'w') f.write('cluster_id\tpurity\n') for i in range(N_tm): f.write('%d\t%g\n' % (i, purity[i])) f.close() for t in range(N_tm): tmp = templates[:, t].toarray().reshape(N_e, N_t).T x, y = tmp.nonzero() nb_loc = len(numpy.unique(y)) if sparse_export: all_positions = numpy.zeros(y.max() + 1, dtype=numpy.int32) all_positions[numpy.unique(y)] = numpy.arange( nb_loc, dtype=numpy.int32) pos = all_positions[y] to_write_sparse[t, x, pos] = tmp[x, y] mapping_sparse[t, numpy.arange(nb_loc)] = numpy.unique(y) else: pos = y to_write_sparse[t, x, pos] = tmp[x, y] if export_all: garbage = io.load_data(params, 'garbage', extension) for t in range(N_tm, N_tm + N_e): elec = t - N_tm spikes = garbage['gspikes'].pop('elec_%d' % elec).astype( numpy.int64) spikes = numpy.random.permutation(spikes)[:100] mapping_sparse[t, 0] = t - N_tm waveform = io.get_stas(params, times_i=spikes, labels_i=np.ones(len(spikes)), src=elec, neighs=[elec], nodes=nodes, mean_mode=True) nb_loc = 1 if sparse_export: to_write_sparse[t, :, 0] = waveform else: to_write_sparse[t, :, elec] = waveform numpy.save(os.path.join(output_path, 'templates'), to_write_sparse) if sparse_export: numpy.save(os.path.join(output_path, 'template_ind'), mapping_sparse) return N_tm def write_pcs(path, params, extension, N_tm, mode=0): spikes = numpy.load(os.path.join(output_path, 'spike_times.npy')) labels = numpy.load(os.path.join(output_path, 'spike_templates.npy')) max_loc_channel = get_max_loc_channel(params, extension) nb_features = params.getint('whitening', 'output_dim') sign_peaks = params.get('detection', 'peaks') nodes, edges = get_nodes_and_edges(params) N_total = params.getint('data', 'N_total') has_support = test_if_support(params, extension) if has_support: supports = io.load_data(params, 'supports', extension) else: inv_nodes = numpy.zeros(N_total, dtype=numpy.int32) inv_nodes[nodes] = numpy.arange(len(nodes)) if export_all: nb_templates = N_tm + N_e else: nb_templates = N_tm pc_features_ind = numpy.zeros((nb_templates, max_loc_channel), dtype=numpy.int32) best_elec = io.load_data(params, 'electrodes', extension) if export_all: best_elec = numpy.concatenate((best_elec, numpy.arange(N_e))) if has_support: for count, support in enumerate(supports): nb_loc = numpy.sum(support) pc_features_ind[count, numpy.arange(nb_loc)] = numpy.where( support == True)[0] else: for count, elec in enumerate(best_elec): nb_loc = len(edges[nodes[elec]]) pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[ nodes[elec]]] if sign_peaks in ['negative', 'both']: basis_proj, basis_rec = io.load_data(params, 'basis') elif sign_peaks in ['positive']: basis_proj, basis_rec = io.load_data(params, 'basis-pos') to_process = numpy.arange(comm.rank, nb_templates, comm.size) all_offsets = numpy.zeros(nb_templates, dtype=numpy.int32) for target in range(nb_templates): if mode == 0: all_offsets[target] = len(numpy.where(labels == target)[0]) elif mode == 1: all_offsets[target] = min( 500, len(numpy.where(labels == target)[0])) all_paddings = numpy.concatenate(([0], numpy.cumsum(all_offsets))) total_pcs = numpy.sum(all_offsets) pc_file = os.path.join(output_path, 'pc_features.npy') pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy') from numpy.lib.format import open_memmap if comm.rank == 0: pc_features = open_memmap(pc_file, shape=(total_pcs, nb_features, max_loc_channel), dtype=numpy.float32, mode='w+') if mode == 1: pc_ids = open_memmap(pc_file_ids, shape=(total_pcs, ), dtype=numpy.int32, mode='w+') comm.Barrier() pc_features = open_memmap(pc_file, mode='r+') if mode == 1: pc_ids = open_memmap(pc_file_ids, mode='r+') to_explore = list(range(comm.rank, nb_templates, comm.size)) if comm.rank == 0: to_explore = get_tqdm_progressbar(params, to_explore) all_idx = numpy.zeros(0, dtype=numpy.int32) for gcount, target in enumerate(to_explore): count = all_paddings[target] if mode == 1: idx = numpy.random.permutation( numpy.where(labels == target)[0])[:500] pc_ids[count:count + len(idx)] = idx elif mode == 0: idx = numpy.where(labels == target)[0] elec = best_elec[target] if has_support: if target >= len(supports): indices = [target - N_tm] else: indices = numpy.where(supports[target])[0] else: indices = inv_nodes[edges[nodes[elec]]] labels_i = target * numpy.ones(len(idx)) times_i = numpy.take(spikes, idx).astype(numpy.int64) sub_data = io.get_stas(params, times_i, labels_i, elec, neighs=indices, nodes=nodes, auto_align=False) pcs = numpy.dot(sub_data, basis_proj) pcs = numpy.swapaxes(pcs, 1, 2) if mode == 0: pc_features[idx, :, :len(indices)] = pcs elif mode == 1: pc_features[count:count + len(idx), :, :len(indices)] = pcs comm.Barrier() if comm.rank == 0: numpy.save(os.path.join(output_path, 'pc_feature_ind'), pc_features_ind.astype( numpy.uint32)) # n_templates, n_loc_chan do_export = True if comm.rank == 0: if os.path.exists(output_path): if not erase_all: do_export = query_yes_no( Fore.WHITE + "Export already made! Do you want to erase everything?", default=None) if do_export: if os.path.exists(os.path.abspath('.phy')): shutil.rmtree(os.path.abspath('.phy')) shutil.rmtree(output_path) if do_export: comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) else: comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) else: do_export = bool( comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)) comm.Barrier() if do_export: #apply_patch_for_similarities(params, extension) if comm.rank == 0: os.makedirs(output_path) print_and_log( ["Exporting data for the phy GUI with %d CPUs..." % nb_cpu], 'info', logger) if params.getboolean('whitening', 'spatial'): whitening_mat = io.load_data( params, 'spatial_whitening').astype(numpy.double) numpy.save(os.path.join(output_path, 'whitening_mat'), whitening_mat) numpy.save(os.path.join(output_path, 'whitening_mat_inv'), numpy.linalg.inv(whitening_mat)) else: numpy.save(os.path.join(output_path, 'whitening_mat'), numpy.eye(N_e)) positions, shanks = generate_mapping(probe) numpy.save(os.path.join(output_path, 'channel_positions'), positions.astype(numpy.double)) numpy.save(os.path.join(output_path, 'channel_shanks'), shanks.astype(numpy.double)) nodes, edges = get_nodes_and_edges(params) numpy.save(os.path.join(output_path, 'channel_map'), nodes.astype(numpy.int32)) write_results(output_path, params, extension) N_tm = write_templates(output_path, params, extension) template_file = h5py.File(file_out_suff + '.templates%s.hdf5' % extension, 'r', libver='earliest') similarities = template_file.get('maxoverlap')[:] template_file.close() norm = N_e * N_t if export_all: to_write = numpy.zeros((N_tm + N_e, N_tm + N_e), dtype=numpy.single) to_write[:N_tm, :N_tm] = (similarities[:N_tm, :N_tm] / norm).astype(numpy.single) else: to_write = (similarities[:N_tm, :N_tm] / norm).astype( numpy.single) numpy.save(os.path.join(output_path, 'similar_templates'), to_write) comm.bcast(numpy.array([N_tm], dtype=numpy.int32), root=0) else: N_tm = int(comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)) comm.Barrier() make_pcs = 2 if comm.rank == 0: if export_pcs == 'prompt': key = '' while key not in ['a', 's', 'n']: print(( Fore.WHITE + "Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o" )) key = input('') else: key = export_pcs if key == 'a': make_pcs = 0 comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) elif key == 's': make_pcs = 1 comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) elif key == 'n': comm.bcast(numpy.array([2], dtype=numpy.int32), root=0) if os.path.exists(os.path.join(output_path, 'pc_features.npy')): os.remove(os.path.join(output_path, 'pc_features.npy')) if os.path.exists( os.path.join(output_path, 'pc_feature_ind.npy')): os.remove(os.path.join(output_path, 'pc_feature_ind.npy')) else: make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) make_pcs = make_pcs[0] comm.Barrier() if make_pcs < 2: write_pcs(output_path, params, extension, N_tm, make_pcs) supported_by_phy = ['raw_binary', 'mcs_raw_binary', 'mda'] file_format = data_file.description gui_params = {} if file_format in supported_by_phy: if not params.getboolean('data', 'overwrite'): gui_params['dat_path'] = r"%s" % params.get( 'data', 'data_file_no_overwrite') else: if params.get('data', 'stream_mode') == 'multi-files': data_file = params.get_data_file(source=True, has_been_created=False) gui_params['dat_path'] = "[" for f in data_file.get_file_names(): gui_params['dat_path'] += 'r"%s", ' % f gui_params['dat_path'] += "]" else: gui_params['dat_path'] = 'r"%s"' % params.get( 'data', 'data_file') else: gui_params['dat_path'] = 'giverandomname.dat' gui_params['n_channels_dat'] = params.nb_channels gui_params['n_features_per_channel'] = 5 gui_params['dtype'] = data_file.data_dtype if 'data_offset' in list(data_file.params.keys()): gui_params['offset'] = data_file.data_offset gui_params['sample_rate'] = params.rate gui_params['dir_path'] = output_path gui_params['hp_filtered'] = True f = open(os.path.join(output_path, 'params.py'), 'w') for key, value in list(gui_params.items()): if key in ['dir_path', 'dtype']: f.write('%s = r"%s"\n' % (key, value)) else: f.write("%s = %s\n" % (key, value)) f.close()
def main(argv=None): if argv is None: argv = sys.argv[1:] header = get_colored_header() parser = argparse.ArgumentParser( description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file') parser.add_argument('-e', '--extension', help='extension to consider for visualization', default='') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) filename = os.path.abspath(args.datafile) extension = args.extension params = CircusParser(filename) if os.path.exists(params.logfile): os.remove(params.logfile) logger = init_logging(params.logfile) logger = logging.getLogger(__name__) mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12") if not mytest: print_and_log( ['You need to update phy-contrib to the latest git version'], 'error', logger) sys.exit(1) if not test_patch_for_similarities(params, extension): print_and_log( ['You should re-export the data because of a fix in 0.6'], 'error', logger) continue_anyway = query_yes_no( Fore.WHITE + "Continue anyway (results may not be fully correct)?", default=None) if not continue_anyway: sys.exit(1) data_file = params.get_data_file() data_dtype = data_file.data_dtype if data_file.params.has_key('data_offset'): data_offset = data_file.data_offset else: data_offset = 0 file_format = data_file.description file_out_suff = params.get('data', 'file_out_suff') if file_format not in supported_by_phy: print_and_log([ "File format %s is not supported by phy. TraceView disabled" % file_format ], 'info', logger) if numpy.iterable(data_file.gain): print_and_log( ['Multiple gains are not supported, using a default value of 1'], 'info', logger) gain = 1 else: if data_file.gain != 1: print_and_log([ "Gain of %g is not supported by phy. Expecting a scaling mismatch" % data_file.gain ], 'info', logger) gain = data_file.gain probe = params.probe if extension != '': extension = '-' + extension output_path = params.get('data', 'file_out_suff') + extension + '.GUI' if not os.path.exists(output_path): print_and_log( ['Data should be first exported with the converting method!'], 'error', logger) else: print_and_log(["Launching the phy GUI..."], 'info', logger) gui_params = {} if file_format in supported_by_phy: if not params.getboolean('data', 'overwrite'): gui_params['dat_path'] = params.get('data', 'data_file_no_overwrite') else: if params.get('data', 'stream_mode') == 'multi-files': data_file = params.get_data_file(source=True, has_been_created=False) gui_params['dat_path'] = ' '.join( data_file.get_file_names()) else: gui_params['dat_path'] = params.get('data', 'data_file') else: gui_params['dat_path'] = 'giverandomname.dat' gui_params['n_channels_dat'] = params.nb_channels gui_params['n_features_per_channel'] = 5 gui_params['dtype'] = data_dtype gui_params['offset'] = data_offset gui_params['sample_rate'] = params.rate gui_params['dir_path'] = output_path gui_params['hp_filtered'] = True f = open(os.path.join(output_path, 'params.py'), 'w') for key, value in gui_params.items(): if key in ['dir_path', 'dat_path', 'dtype']: f.write('%s = "%s"\n' % (key, value)) else: f.write("%s = %s\n" % (key, value)) f.close() os.chdir(output_path) create_app() controller = TemplateController(**gui_params) gui = controller.create_gui() gui.show() run_app() gui.close() del gui
def main(argv=None): if argv is None: argv = sys.argv[1:] parallel_hdf5 = h5py.get_config().mpi user_path = pjoin(os.path.expanduser('~'), 'spyking-circus') tasks_list = None if not os.path.exists(user_path): os.makedirs(user_path) try: import cudamat as cmt cmt.init() HAVE_CUDA = True except Exception: HAVE_CUDA = False all_steps = [ 'whitening', 'clustering', 'fitting', 'gathering', 'extracting', 'filtering', 'converting', 'deconverting', 'benchmarking', 'merging', 'validating', 'thresholding' ] config_file = os.path.abspath(pkg_resources.resource_filename('circus', 'config.params')) header = get_colored_header() header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str(psutil.cpu_count()) + '\n' # header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str(HAVE_CUDA) + '\n' header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str(parallel_hdf5) + '\n' do_upgrade = '' if not SHARED_MEMORY: do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]' header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str(SHARED_MEMORY) + do_upgrade + '\n' header += '\n' header += Fore.GREEN + "##################################################################" header += Fore.RESET method_help = '''by default, all steps are performed, but a subset x,y can be done. Steps are: - filtering - whitening - clustering - fitting - merging [with or without a GUI for meta merging] - (extra) converting [export results to phy format] - (extra) thresholding [to get MUA activity only] - (extra) deconverting [import results from phy format] - (extra) gathering [force collection of results] - (extra) extracting [get templates from spike times] - (extra) benchmarking [with -o and -t] - (extra) validating [to compare performance with GT neurons]''' parser = argparse.ArgumentParser(description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file (or a list of commands if batch mode)') parser.add_argument('-i', '--info', help='list the file formats supported by SpyKING CIRCUS', action='store_true') parser.add_argument('-m', '--method', default='filtering,whitening,clustering,fitting,merging', help=method_help) parser.add_argument('-c', '--cpu', type=int, default=max(1, int(psutil.cpu_count()/2)), help='number of CPU') # parser.add_argument('-g', '--gpu', type=int, default=0, help='number of GPU') parser.add_argument('-H', '--hostfile', help='hostfile for MPI', default=pjoin(user_path, 'circus.hosts')) parser.add_argument('-b', '--batch', help='datafile is a list of commands to launch, in a batch mode', action='store_true') parser.add_argument('-p', '--preview', help='GUI to display the first second filtered with thresholds', action='store_true') parser.add_argument('-r', '--result', help='GUI to display the results on top of raw data', action='store_true') parser.add_argument('-s', '--second', type=int, default=0, help='If preview mode, begining of the preview [in s]') parser.add_argument('-e', '--extension', help='extension to consider for merging, converting and deconverting', default='None') parser.add_argument('-o', '--output', help='output file [for generation of synthetic benchmarks]') parser.add_argument('-t', '--type', help='benchmark type', choices=['fitting', 'clustering', 'synchrony']) if len(argv) == 0: parser.print_help() sys.exit(0) args = parser.parse_args(argv) steps = args.method.split(',') for step in steps: if step not in all_steps: print_error(['The method "%s" is not recognized' % step]) sys.exit(0) # To save some typing later nb_gpu = 0 (nb_cpu, hostfile, batch, preview, result, extension, output, benchmark, info, second) = \ (args.cpu, args.hostfile, args.batch, args.preview, args.result, args.extension, args.output, args.type, args.info, args.second) filename = os.path.abspath(args.datafile) real_file = filename f_next, extens = os.path.splitext(filename) if info: if args.datafile.lower() in __supported_data_files__: filename = 'tmp' if len(__supported_data_files__[args.datafile.lower()].extension) > 0: filename += __supported_data_files__[args.datafile.lower()].extension[0] __supported_data_files__[args.datafile.lower()](filename, {}, is_empty=True)._display_requirements_() else: print_and_log([ '', 'To get info on any particular file format, do:', '>> spyking-circus file_format -i', '' ], 'default') print_and_log(list_all_file_format()) sys.exit(0) if extens == '.params': print_error(['You should launch the code on the data file!']) sys.exit(0) file_params = f_next + '.params' if not os.path.exists(file_params) and not batch: print(Fore.RED + 'The parameter file %s is not present!' % file_params) create_params = query_yes_no(Fore.WHITE + "Do you want SpyKING CIRCUS to create a parameter file?") if create_params: print(Fore.WHITE + "Creating %s" % file_params) print(Fore.WHITE + "Fill it properly before launching the code! (see documentation)") print_info(['Keep in mind that filtering is performed on site, so please', 'be sure to keep a copy of your data elsewhere']) shutil.copyfile(config_file, file_params) sys.exit(0) elif batch: tasks_list = filename if not batch: file_params = f_next + '.params' if not os.path.exists(file_params): print_and_log(["%s does not exist" % file_params], 'error') sys.exit(0) import ConfigParser as configparser parser = configparser.ConfigParser() myfile = open(file_params, 'r') lines = myfile.readlines() myfile.close() myfile = open(file_params, 'w') for l in lines: myfile.write(l.replace('\t', '')) myfile.close() parser.read(file_params) for section in CircusParser.__all_sections__: if parser.has_section(section): for (key, value) in parser.items(section): parser.set(section, key, value.split('#')[0].rstrip()) else: parser.add_section(section) try: use_output_dir = parser.get('data', 'output_dir') != '' except Exception: use_output_dir = False if use_output_dir: path = os.path.abspath(os.path.expanduser(parser.get('data', 'output_dir'))) file_out = os.path.join(path, os.path.basename(f_next)) if not os.path.exists(file_out): os.makedirs(file_out) else: file_out = f_next logfile = file_out + '.log' if os.path.exists(logfile): os.remove(logfile) logger = init_logging(logfile) params = CircusParser(filename) data_file = params.get_data_file(source=True, has_been_created=False) overwrite = params.getboolean('data', 'overwrite') file_format = params.get('data', 'file_format') if overwrite: support_parallel_write = data_file.parallel_write is_writable = data_file.is_writable else: support_parallel_write = __supported_data_files__['raw_binary'].parallel_write is_writable = __supported_data_files__['raw_binary'].is_writable if preview: print_and_log(['Preview mode, showing only seconds [%d-%d] of the recording' % (second, second+1)], 'info', logger) tmp_path_loc = os.path.join(os.path.abspath(params.get('data', 'file_out')), 'tmp') if not os.path.exists(tmp_path_loc): os.makedirs(tmp_path_loc) filename = os.path.join(tmp_path_loc, 'preview.dat') f_next, extens = os.path.splitext(filename) preview_params = f_next + '.params' shutil.copyfile(file_params, preview_params) steps = ['filtering', 'whitening'] chunk_size = int(params.rate) data_file.open() nb_chunks, _ = data_file.analyze(chunk_size) if nb_chunks <= (second + 1): print_and_log(['Recording is too short to display seconds [%d-%d]' % (second, second+1)]) sys.exit(0) local_chunk = data_file.get_snippet(int(second*params.rate), int(1.2*chunk_size)) description = data_file.get_description() data_file.close() new_params = CircusParser(filename, create_folders=False) new_params.write('data', 'chunk_size', '1') new_params.write('data', 'file_format', 'raw_binary') new_params.write('data', 'data_dtype', 'float32') new_params.write('data', 'data_offset', '0') new_params.write('data', 'dtype_offset', '0') new_params.write('data', 'stream_mode', 'None') new_params.write('data', 'overwrite', 'True') new_params.write('triggers', 'ignore_times', 'False') new_params.write('data', 'sampling_rate', str(params.rate)) new_params.write('whitening', 'safety_time', '0') new_params.write('clustering', 'safety_time', '0') new_params.write('whitening', 'chunk_size', '1') new_params.write('data', 'preview_path', params.file_params) new_params.write('data', 'output_dir', '') description['data_dtype'] = 'float32' description['dtype_offset'] = 0 description['data_offset'] = 0 description['gain'] = 1. new_params = CircusParser(filename) data_file_out = new_params.get_data_file(is_empty=True, params=description) support_parallel_write = data_file_out.parallel_write is_writable = data_file_out.is_writable data_file_out.allocate(shape=local_chunk.shape, data_dtype=numpy.float32) data_file_out.open('r+') data_file_out.set_data(0, local_chunk) data_file_out.close() if tasks_list is not None: with open(tasks_list, 'r') as f: for line in f: if len(line) > 0: subprocess.check_call(['spyking-circus'] + line.replace('\n', '').split(" ")) else: print_and_log(['Config file: %s' % (f_next + '.params')], 'debug', logger) print_and_log(['Data file : %s' % filename], 'debug', logger) print(get_colored_header()) print(Fore.GREEN + "File : " + Fore.CYAN + real_file) if preview: print(Fore.GREEN + "Steps : " + Fore.CYAN + "preview mode") elif result: print(Fore.GREEN + "Steps : " + Fore.CYAN + "result mode") else: print(Fore.GREEN + "Steps : " + Fore.CYAN + ", ".join(steps)) # print Fore.GREEN + "GPU detected : ", Fore.CYAN + str(HAVE_CUDA) print(Fore.GREEN + "Number of CPU : " + Fore.CYAN + str(nb_cpu) + "/" + str(psutil.cpu_count())) # if HAVE_CUDA: # print Fore.GREEN + "Number of GPU : ", Fore.CYAN + str(nb_gpu) print(Fore.GREEN + "Parallel HDF5 : " + Fore.CYAN + str(parallel_hdf5)) do_upgrade = '' use_shared_memory = get_shared_memory_flag(params) if not SHARED_MEMORY: do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]' print(Fore.GREEN + "Shared memory : " + Fore.CYAN + str(use_shared_memory) + do_upgrade) print(Fore.GREEN + "Hostfile : " + Fore.CYAN + hostfile) print("") print(Fore.GREEN + "##################################################################") print("") print(Fore.RESET) # Launch the subtasks subtasks = [('filtering', 'mpirun'), ('whitening', 'mpirun'), ('clustering', 'mpirun'), ('fitting', 'mpirun'), ('extracting', 'mpirun'), ('gathering', 'python'), ('converting', 'mpirun'), ('deconverting', 'mpirun'), ('benchmarking', 'mpirun'), ('merging', 'mpirun'), ('validating', 'mpirun'), ('thresholding', 'mpirun')] # if HAVE_CUDA and nb_gpu > 0: # use_gpu = 'True' # else: use_gpu = 'False' time = data_stats(params) / 60.0 if preview: params = new_params if nb_cpu < psutil.cpu_count(): if use_gpu != 'True' and not result: print_and_log(['Using only %d out of %d local CPUs available (-c to change)' % (nb_cpu, psutil.cpu_count())], 'info', logger) if params.getboolean('detection', 'matched-filter') and not params.getboolean('clustering', 'smart_search'): print_and_log(['Smart Search should be activated for matched filtering'], 'info', logger) if time > 30 and not params.getboolean('clustering', 'smart_search'): print_and_log(['Smart Search should be activated for long recordings'], 'info', logger) n_edges = get_averaged_n_edges(params) if n_edges > 100 and not params.getboolean('clustering', 'compress'): print_and_log(['Template compression is highly recommended based on parameters'], 'info', logger) if not result: for subtask, command in subtasks: if subtask in steps: if command == 'python': # Directly call the launcher try: circus.launch(subtask, filename, nb_cpu, nb_gpu, use_gpu) except: print_and_log(['Step "%s" failed!' % subtask], 'error', logger) sys.exit(0) elif command == 'mpirun': # Use mpirun to make the call mpi_args = gather_mpi_arguments(hostfile, params) one_cpu = False if subtask in ['filtering', 'benchmarking'] and not is_writable: if not preview and overwrite: print_and_log(['The file format %s is read only!' % file_format, 'You should set overwite to False, to create a copy of the data.', 'However, note that if you have streams, informations on times', 'will be discarded'], 'info', logger) sys.exit(0) if subtask in ['filtering'] and not support_parallel_write and (args.cpu > 1): print_and_log(['No parallel writes for %s: only 1 node used for %s' %(file_format, subtask)], 'info', logger) nb_tasks = str(1) one_cpu = True else: if subtask != 'fitting': nb_tasks = str(args.cpu) else: # if use_gpu == 'True': # nb_tasks = str(args.gpu) # else: nb_tasks = str(args.cpu) if subtask == 'benchmarking': if (output is None) or (benchmark is None): print_and_log(["To generate synthetic datasets, you must provide output and type"], 'error', logger) sys.exit(0) mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, output, benchmark ] elif subtask in ['merging', 'converting']: mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, extension ] elif subtask in ['deconverting']: nb_tasks = str(1) nb_cpu = 1 mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, extension ] else: mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, str(one_cpu) ] print_and_log(['Launching task %s' % subtask], 'debug', logger) print_and_log(['Command: %s' % str(mpi_args)], 'debug', logger) try: subprocess.check_call(mpi_args) except subprocess.CalledProcessError as e: print_and_log(['Step "%s" failed for reason %s!' % (subtask, e)], 'error', logger) sys.exit(0) if preview or result: from circus.shared import gui import pylab try: from PyQt5.QtWidgets import QApplication except ImportError: from matplotlib.backends import qt_compat use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE if use_pyside: from PySide.QtGui import QApplication else: from PyQt4.QtGui import QApplication app = QApplication([]) try: pylab.style.use('ggplot') except Exception: pass if preview: print_and_log(['Launching the preview GUI...'], 'debug', logger) mygui = gui.PreviewGUI(new_params) shutil.rmtree(tmp_path_loc) elif result: data_file = params.get_data_file() print_and_log(['Launching the result GUI...'], 'debug', logger) mygui = gui.PreviewGUI(params, show_fit=True) sys.exit(app.exec_())
def main(argv=None): if argv is None: argv = sys.argv[1:] header = get_colored_header() header += '''Utility to group files within several folders into a single virtual folder, such that they can be processed together with the multi-files mode. If you want to also process .dead or .trig files in order to later on concatenate artefacts, please use the -d or -t options ''' parser = argparse.ArgumentParser(description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('folders', help='text file with the list of folders to consider') parser.add_argument('extension', help='file extension to consider within folders') parser.add_argument('-o', '--output', help='name of the output folder [default is output]', default='output') parser.add_argument('-d', '--dead', help='Search for all .dead files', action='store_true') parser.add_argument('-t', '--trig', help='Search for all .trig files', action='store_true') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) folders_file = os.path.abspath(args.folders) output = os.path.abspath(args.output) extension = args.extension filename, ext = os.path.splitext(os.path.basename(folders_file)) logger = init_logging(filename + '.log') logger = logging.getLogger(__name__) if not os.path.exists(folders_file): print_and_log(['The folder file %s does not exists!' %folders_file], 'error', logger) sys.exit(0) try: folders = [] myfile = open(folders_file, 'r') lines = myfile.readlines() myfile.close() for l in lines: folders += [os.path.abspath(l.strip())] except Exception: print_and_log(['Check the syntax of the folder file'], 'error', logger) sys.exit(0) do_folders = True if os.path.exists(output): do_folders = query_yes_no(Fore.WHITE + "Folder %s already exists! Do you want to erase everything?" %output, default=None) if not do_folders: sys.exit(0) else: shutil.rmtree(output) os.makedirs(output) for count, folder in enumerate(folders): files = os.listdir(folder) for file in files: _, ext = os.path.splitext(file) ext = ext.strip('.') if (ext.lower() == extension.lower()) or (args.dead and ext.lower() == 'dead') or (args.trig and ext.lower()== 'trig'): original_file = os.path.join(folder, file) linked_file = os.path.join(output, 'sc_{c}_{f}'.format(c=count, f=os.path.basename(original_file))) if not os.path.exists(linked_file): os.symlink(original_file, linked_file) else: os.symlink(original_file, linked_file)
def main(params, nb_cpu, nb_gpu, use_gpu, extension): _ = init_logging(params.logfile) logger = logging.getLogger('circus.merging') file_out_suff = params.get('data', 'file_out_suff') erase_all = params.getboolean('merging', 'erase_all') extension_in = extension extension_out = '-merged' # Erase previous results (if user agrees). if comm.rank == 0: existing_file_paths = [ file_path for file_path in [ file_out_suff + ".%s%s.hdf5" % (file_id, extension_out) for file_id in ['templates', 'clusters', 'result'] ] if os.path.isfile(file_path) ] existing_directory_path = [ directory_path for directory_path in [file_out_suff + "%s.GUI" % extension_out] if os.path.isdir(directory_path) ] if len(existing_file_paths) > 0 or len(existing_directory_path) > 0: if not erase_all: erase = query_yes_no( "Merging already done! Do you want to erase previous merging results?", default=None) else: erase = True if erase: for path in existing_file_paths: os.remove(path) if comm.rank == 0: print_and_log(["Removed file %s" % path], 'debug', logger) for path in existing_directory_path: shutil.rmtree(path) if comm.rank == 0: print_and_log(["Removed directory %s" % path], 'debug', logger) comm.Barrier() if params.getfloat('merging', 'auto_mode') == 0: from sys import platform if not platform == 'win32': if not ('DISPLAY' in os.environ and re.search(":\d", os.environ['DISPLAY']) != None): print_and_log( ['Preview mode can not be used, check DISPLAY variable'], 'error', logger) sys.exit(0) if comm.rank == 0: try: from PyQt5.QtWidgets import QApplication except ImportError: from matplotlib.backends import qt_compat use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE if use_pyside: from PySide.QtGui import QApplication else: from PyQt4.QtGui import QApplication app = QApplication([]) try: pylab.style.use('ggplot') except Exception: pass else: app = None else: app = None if comm.rank == 0: print_and_log(['Launching the merging GUI...'], 'debug', logger) _ = gui.MergeWindow(params, app, extension_in, extension_out) sys.exit(app.exec_())
def main(params, nb_cpu, nb_gpu, use_gpu, extension): assert comm.rank == 0 input_extension = extension logger = init_logging(params.logfile) logger = logging.getLogger('circus.deconverting') # Retrieve parameters. input_path = params.get('data', 'file_out_suff') + input_extension + '.GUI' output_path = params.get('data', 'file_out_suff') output_extension = '-deconverted' clusters_path = output_path + '.clusters{}.hdf5'.format(output_extension) templates_path = output_path + '.templates{}.hdf5'.format(output_extension) result_path = output_path + '.result{}.hdf5'.format(output_extension) # Check if input path exists. if not os.path.isdir(input_path): print_and_log([ "Can't find directory: {}".format(input_path), "You must first export your results into the phy format (use the converting method)." ], 'error', logger) sys.exit(0) # Check if results are already present. if os.path.isfile(clusters_path) \ and os.path.isfile(templates_path) \ and os.path.isfile(result_path): print_and_log([ "Phy results already imported.", "Delete the following files to run the deconversion again:", " - {}".format(result_path), " - {}".format(templates_path), " - {}".format(clusters_path), ], 'error', logger) if query_yes_no("Do you want to delete these files?"): os.remove(result_path) os.remove(templates_path) os.remove(clusters_path) else: sys.exit(0) # Check if results are partially present. if os.path.isfile(clusters_path) \ or os.path.isfile(templates_path) \ or os.path.isfile(result_path): print_and_log([ "Phy results partially imported.", "Delete the following files to be able to run the deconversion:", ] + [ " - {}".format(path) for path in [result_path, templates_path, clusters_path] if os.path.isfile(path) ], 'error', logger) if query_yes_no("Do you want to delete these files?"): for path in [result_path, templates_path, clusters_path]: if os.path.isfile(path): os.remove(path) else: sys.exit(0) # Log introduction message. message = "Importing data from the phy GUI with {} CPUs...".format(nb_cpu) print_and_log([message], 'info', logger) # Read phy results. phy_results = load_phy_results(input_path) spike_templates = phy_results['spike_templates'] spike_clusters = phy_results['spike_clusters'] cluster_group = phy_results['cluster_group'] templates = phy_results['templates'] # print_and_log(["{}".format(phy_results)], 'debug', logger) # Read spyking-circus results. templates_input_path = output_path + ".templates{}.hdf5".format( input_extension) templates_input_file = h5py.File(templates_input_path, mode='r', libver='earliest') overlaps = templates_input_file.get('maxoverlap')[:] try: lag = templates_input_file.get('maxlag')[:] except TypeError: # i.e. 'maxlag' not in HDF5 file. lag = np.zeros(overlaps.shape, dtype=np.int32) shape = templates_input_file.get('temp_shape')[:] # Set correct lag option. correct_lag = True # Determine association map between spike templates and spike clusters. templates_to_clusters = {} for spike_template, spike_cluster in zip(spike_templates, spike_clusters): templates_to_clusters[spike_template] = spike_cluster # Determine association map between spike cluster and default spike template. clusters_to_templates = {} for spike_cluster in cluster_group: clusters_to_templates[spike_cluster] = None # # TODO remove templates without spikes. # print_and_log([ # TODO remove. # "Removing templates without spikes..." # ], 'info', logger) # electrodes = io.load_data(params, 'electrodes', extension=input_extension) # TODO remove duplicate. # clusters = io.load_data(params, 'clusters', extension=input_extension) # TODO remove duplicate # for spike_template, _ in templates_to_clusters.items(): # # Retrieve the prefered electrode. # elec_ic = electrodes[spike_template] # # Retrieve template index among templates with same prefered electrodeself. # first_index = np.where(electrodes == elec_ic)[0][0] # nic = spike_template - first_index # # Retrieve the cluster label. # label = 'clusters_{}'.format(elec_ic) # # Select the points labelled by the clustering. # mask = clusters[label] > -1 # # Retrieve the labels used by the clustering. # tmp = np.unique(clusters[label][mask]) # # Retrieve the number of points labelled for both templates. # cluster_label = tmp[nic] # elements = np.where(clusters[label] == cluster_label)[0] # # ... # if len(elements) == 0: # print_and_log([ # "template {} has no spike".format(spike_template) # ], 'info', logger) # raise NotImplementedError to_merge = [] to_remove = [] # Do all the merges. old_results = io.load_data(params, 'results', extension=input_extension) electrodes = io.load_data(params, 'electrodes', extension=input_extension) clusters = io.load_data(params, 'clusters', extension=input_extension) for spike_template, spike_cluster in templates_to_clusters.items(): spike_group = cluster_group[spike_cluster] if spike_group in ['good', 'unsorted']: if clusters_to_templates[spike_cluster] is None: clusters_to_templates[spike_cluster] = spike_template else: # Retrieve pair of templates to merge. default_spike_template = clusters_to_templates[spike_cluster] one_merge = [default_spike_template, spike_template] # Retrieve the prefered electrode for both template. elec_ic1 = electrodes[one_merge[0]] elec_ic2 = electrodes[one_merge[1]] # Retrieve template index among templates with same prefered electrode for both templates. first_index1 = np.where(electrodes == elec_ic1)[0][0] first_index2 = np.where(electrodes == elec_ic2)[0][0] nic1 = one_merge[0] - first_index1 nic2 = one_merge[1] - first_index2 # Retrieve the cluster label for both templates. label1 = 'clusters_{}'.format(elec_ic1) label2 = 'clusters_{}'.format(elec_ic2) # Select the points labelled by the clustering for both templates. mask1 = clusters[label1] > -1 mask2 = clusters[label2] > -1 # Retrieve the labels used by the clustering for both templates. tmp1 = np.unique(clusters[label1][mask1]) tmp2 = np.unique(clusters[label2][mask2]) # Retrieve the number of points labelled for both templates. cluster_label1 = tmp1[nic1] cluster_label2 = tmp2[nic2] elements1 = np.where(clusters[label1] == cluster_label1)[0] elements2 = np.where(clusters[label2] == cluster_label2)[0] # Determine index to keep and index to delete. if len(elements1) > len(elements2): to_delete = one_merge[1] to_keep = one_merge[0] elec = elec_ic2 elements = elements2 else: to_delete = one_merge[0] to_keep = one_merge[1] elec = elec_ic1 elements = elements1 # print_and_log([ # "one_merge: {}".format(one_merge), # "elec_ic1, elec_ic2: {}, {}".format(elec_ic1, elec_ic2), # "nic1, nic2: {}, {}".format(nic1, nic2), # "tmp1, tmp2: {}, {}".format(tmp1, tmp2), # "cluster_label1, cluster_label2: {}, {}".format(cluster_label1, cluster_label2), # "to_keep, to_delete: {}, {}".format(to_keep, to_delete) # ] , 'debug', logger # ) # Merge templates (if necessary). if to_keep != to_delete: key1 = 'temp_{}'.format(to_keep) key2 = 'temp_{}'.format(to_delete) amplitudes1 = old_results['amplitudes'][key1] amplitudes2 = old_results['amplitudes'][key2] old_results['amplitudes'][key1] = np.vstack( (amplitudes1, amplitudes2)) spiketimes1 = old_results['spiketimes'][key1] spiketimes2 = old_results['spiketimes'][key2] if correct_lag: spiketimes2 = spiketimes2.astype(np.int64) spiketimes2 += lag[to_keep, to_delete] spiketimes2 = spiketimes2.astype(np.uint32) old_results['spiketimes'][key1] = np.concatenate( (spiketimes1, spiketimes2)) indices = np.argsort(old_results['spiketimes'][key1]) old_results['amplitudes'][key1] = old_results[ 'amplitudes'][key1][indices] old_results['spiketimes'][key1] = old_results[ 'spiketimes'][key1][indices] old_results['amplitudes'].pop(key2) old_results['spiketimes'].pop(key2) # Update internal variables. clusters_to_templates[spike_cluster] = to_keep to_merge.append((to_keep, to_delete)) elif spike_group in ['mua', 'noise']: to_remove.append(spike_template) else: message = "Unexpected group value: {}".format(spike_group) raise ValueError(message) # Remove unmentioned templates (e.g. without any fitted spike). old_templates = load_data(params, 'templates', extension=input_extension) initial_nb_templates = old_templates.shape[1] // 2 all_spike_templates = set(range(0, initial_nb_templates)) mentioned_spike_templates = set(templates_to_clusters.keys()) unmentioned_spike_templates = list(all_spike_templates - mentioned_spike_templates) # print_and_log(["unmentioned templates: {}".format(unmentioned_spike_templates)], 'info', logger) to_remove.extend(unmentioned_spike_templates) if to_merge == []: to_merge = np.zeros((0, 2), dtype=np.int) else: to_merge = np.array(to_merge) to_merge = to_merge[np.lexsort((to_merge[:, 1], to_merge[:, 0])), :] to_remove.sort() # Log some information. nb_merges = to_merge.shape[0] nb_removals = len(to_remove) final_nb_templates = initial_nb_templates - nb_merges - nb_removals print_and_log([ "Manual sorting with the Python GUI (i.e. phy):", " initial number of templates: {}".format(initial_nb_templates), " number of merges: {}".format(nb_merges), " number of removals: {}".format(nb_removals), " final number of templates: {}".format(final_nb_templates), ], 'info', logger) # Slice templates. to_keep = slice_templates(params, to_merge=to_merge, to_remove=to_remove, extension=output_extension, input_extension=extension) # print_and_log([ # "to_merge (passed to slice_templates: {}".format(to_merge), # "to_remove (passed to slice_templates: {}".format(to_remove), # "to_keep (returned be slice_templates): {}".format(to_keep), # ], 'info', logger) # Slice clusters. light = True dataname = 'clusters-light' if light else 'clusters' clusters = io.load_data(params, dataname, extension=extension) slice_clusters(params, clusters, to_merge=to_merge, to_remove=to_remove, extension=output_extension, input_extension=extension, light=light, method='new') # Finalize result. # nb_templates = templates.shape[0] # template_indices = np.arange(0, nb_templates) # to_delete = list(to_remove) # for one_merge in to_merge: # to_delete.append(one_merge[1]) # to_keep = set(np.unique(template_indices)) - set(to_delete) # to_keep = np.array(list(to_keep)) # to_keep = np.sort(to_keep) # # Would be correct if we could sort 'to_keep' in 'slice_templates'. # print_and_log([ # "to_keep: {}".format(to_keep) # ], 'idebug', logger) new_results = { 'spiketimes': {}, 'amplitudes': {}, } for k, template_index in enumerate(to_keep): old_key = 'temp_{}'.format(template_index) new_key = 'temp_{}'.format(k) new_results['spiketimes'][new_key] = old_results['spiketimes'].pop( old_key) new_results['amplitudes'][new_key] = old_results['amplitudes'].pop( old_key) # Check if the number of spikes is not equal to 0. nb_spikes = len(new_results['spiketimes'][new_key]) if nb_spikes == 0: print_and_log( ["{} - template {} has no spikes".format(k, template_index)], 'error', logger) keys = ['spiketimes', 'amplitudes'] # TODO add support for [fitting] collect_all=True (not supported). # Save new result to output file. result_output_path = output_path + ".result{}.hdf5".format( output_extension) result_output_file = h5py.File(result_output_path, mode='w', libver='earliest') for key in keys: result_output_file.create_group(key) for temp in new_results[key].keys(): tmp_path = "{}/{}".format(key, temp) result_output_file.create_dataset(tmp_path, data=new_results[key][temp]) result_output_file.close() # Add additional information to templates file. templates_output_path = output_path + ".templates{}.hdf5".format( output_extension) templates_output_file = h5py.File(templates_output_path, mode='r+', libver='earliest') new_shape = (len(to_keep), len(to_keep)) version = templates_output_file.create_dataset( 'version', data=np.array(circus.__version__.split('.'), dtype=np.int32)) maxoverlaps = templates_output_file.create_dataset('maxoverlap', shape=new_shape, dtype=np.float32) maxlag = templates_output_file.create_dataset('maxlag', shape=new_shape, dtype=np.int32) for k, index in enumerate(to_keep): maxoverlaps[k, :] = overlaps[index, to_keep] maxlag[k, :] = lag[index, to_keep] templates_output_file.close() # Log conclusion message. message = "Data from the phy GUI imported." print_and_log([message], 'info', logger) return