def slice_clusters(comm, params, result, to_remove=[], to_merge=[], extension='', light=False): import h5py, shutil file_out_suff = params.get('data', 'file_out_suff') N_e = params.getint('data', 'N_e') if comm.rank == 0: print_and_log(['Node 0 is slicing clusters'], 'debug', params) if to_merge != []: for count in xrange(len(to_merge)): remove = to_merge[count][1] to_remove += [remove] all_elements = [[] for i in xrange(N_e)] for target in numpy.unique(to_remove): elec = result['electrodes'][target] nic = target - numpy.where(result['electrodes'] == elec)[0][0] mask = result['clusters_' + str(elec)] > -1 tmp = numpy.unique(result['clusters_' + str(elec)][mask]) all_elements[elec] += list(numpy.where(result['clusters_' + str(elec)] == tmp[nic])[0]) for elec in xrange(N_e): if not light: result['data_' + str(elec)] = numpy.delete(result['data_' + str(elec)], all_elements[elec], axis=0) result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec]) result['times_' + str(elec)] = numpy.delete(result['times_' + str(elec)], all_elements[elec]) result['peaks_' + str(elec)] = numpy.delete(result['peaks_' + str(elec)], all_elements[elec]) else: result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec]) myfile = h5py.File(file_out_suff + '.clusters.hdf5', 'r', libver='latest') data = myfile.get('data_' + str(elec))[:] result['data_' + str(elec)] = numpy.delete(data, all_elements[elec], axis=0) data = myfile.get('times_' + str(elec))[:] result['times_' + str(elec)] = numpy.delete(data, all_elements[elec]) data = myfile.get('peaks_' + str(elec))[:] result['peaks_' + str(elec)] = numpy.delete(data, all_elements[elec]) myfile.close() result['electrodes'] = numpy.delete(result['electrodes'], numpy.unique(to_remove)) cfile = h5py.File(file_out_suff + '.clusters-new.hdf5', 'w', libver='latest') to_write = ['data_', 'clusters_', 'times_', 'peaks_'] for ielec in xrange(N_e): write_datasets(cfile, to_write, result, ielec) write_datasets(cfile, ['electrodes'], result) cfile.close() if os.path.exists(file_out_suff + '.clusters%s.hdf5' %extension): os.remove(file_out_suff + '.clusters%s.hdf5' %extension) shutil.move(file_out_suff + '.clusters-new.hdf5', file_out_suff + '.clusters%s.hdf5' %extension) comm.Barrier()
def main(argv=None): if argv is None: argv = sys.argv[1:] gheader = Fore.GREEN + get_header() header = gheader + Fore.RESET parser = argparse.ArgumentParser( description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file') parser.add_argument('-e', '--extension', help='extension to consider for slicing results', default='') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) filename = os.path.abspath(args.datafile) extension = args.extension params = circus.shared.utils.io.load_parameters(filename) file_out_suff = params.get('data', 'file_out_suff') if not params.get('data', 'multi-files'): print_and_log(['Not a multi-file!'], 'error', params) sys.exit(0) to_process = circus.shared.files.get_multi_files(params) result = circus.shared.files.get_results(params, extension=extension) times = circus.shared.files.data_stats(params, show=False, export_times=True) sub_results = slice_result(result, times) for count, result in enumerate(sub_results): keys = ['spiketimes', 'amplitudes'] mydata = h5py.File(file_out_suff + '.result%s_%d.hdf5' % (extension, count), 'w', libver='latest') for key in keys: mydata.create_group(key) for temp in result[key].keys(): tmp_path = '%s/%s' % (key, temp) mydata.create_dataset(tmp_path, data=result[key][temp]) mydata.close()
def slice_templates(comm, params, to_remove=[], to_merge=[], extension=''): import shutil, h5py file_out_suff = params.get('data', 'file_out_suff') if comm.rank == 0: print_and_log(['Node 0 is slicing templates'], 'debug', params) old_templates = load_data(params, 'templates') old_limits = load_data(params, 'limits') N_e = params.getint('data', 'N_e') N_t = params.getint('data', 'N_t') x, N_tm = old_templates.shape norm_templates = load_data(params, 'norm-templates') if to_merge != []: for count in xrange(len(to_merge)): remove = to_merge[count][1] to_remove += [remove] all_templates = set(numpy.arange(N_tm//2)) to_keep = numpy.array(list(all_templates.difference(to_remove))) positions = numpy.arange(len(to_keep)) local_keep = to_keep[positions] templates = scipy.sparse.lil_matrix((N_e*N_t, 2*len(to_keep)), dtype=numpy.float32) hfile = h5py.File(file_out_suff + '.templates-new.hdf5', 'w', libver='latest') norms = hfile.create_dataset('norms', shape=(2*len(to_keep), ), dtype=numpy.float32, chunks=True) limits = hfile.create_dataset('limits', shape=(len(to_keep), 2), dtype=numpy.float32, chunks=True) for count, keep in zip(positions, local_keep): templates[:, count] = old_templates[:, keep] templates[:, count + len(to_keep)] = old_templates[:, keep + N_tm//2] norms[count] = norm_templates[keep] norms[count + len(to_keep)] = norm_templates[keep + N_tm//2] if to_merge == []: new_limits = old_limits[keep] else: subset = numpy.where(to_merge[:, 0] == keep)[0] if len(subset) > 0: idx = numpy.unique(to_merge[subset].flatten()) ratios = norm_templates[idx]/norm_templates[keep] new_limits = [numpy.min(ratios*old_limits[idx][:, 0]), numpy.max(ratios*old_limits[idx][:, 1])] else: new_limits = old_limits[keep] limits[count] = new_limits templates = templates.tocoo() hfile.create_dataset('temp_x', data=templates.row) hfile.create_dataset('temp_y', data=templates.col) hfile.create_dataset('temp_data', data=templates.data) hfile.create_dataset('temp_shape', data=numpy.array([N_e, N_t, 2*len(to_keep)], dtype=numpy.int32)) hfile.close() if os.path.exists(file_out_suff + '.templates%s.hdf5' %extension): os.remove(file_out_suff + '.templates%s.hdf5' %extension) shutil.move(file_out_suff + '.templates-new.hdf5', file_out_suff + '.templates%s.hdf5' %extension) comm.Barrier()
def main(argv=None): if argv is None: argv = sys.argv[1:] gheader = Fore.GREEN + get_header() header = gheader + Fore.RESET parser = argparse.ArgumentParser( description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file') parser.add_argument('-e', '--extension', help='extension to consider for visualization', default='') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) filename = os.path.abspath(args.datafile) extension = args.extension params = circus.shared.utils.io.load_parameters(filename) sampling_rate = params.getint('data', 'sampling_rate') data_dtype = params.get('data', 'data_dtype') gain = 1 file_out_suff = params.get('data', 'file_out_suff') data_offset = params.getint('data', 'data_offset') probe = read_probe(params) if extension != '': extension = '-' + extension def generate_matlab_mapping(probe): p = {} positions = [] nodes = [] for key in probe['channel_groups'].keys(): p.update(probe['channel_groups'][key]['geometry']) nodes += probe['channel_groups'][key]['channels'] positions += [ p[channel] for channel in probe['channel_groups'][key]['channels'] ] idx = numpy.argsort(nodes) positions = numpy.array(positions)[idx] t = tempfile.NamedTemporaryFile().name + '.hdf5' cfile = h5py.File(t, 'w') to_write = { 'positions': positions / 10., 'permutation': numpy.sort(nodes), 'nb_total': numpy.array([probe['total_nb_channels']]) } write_datasets(cfile, to_write.keys(), to_write) cfile.close() return t mapping = generate_matlab_mapping(probe) filename = params.get('data', 'data_file') gui_params = [ sampling_rate, os.path.abspath(file_out_suff), '%s.mat' % extension, mapping, 2, data_dtype, data_offset, gain, filename ] gui_file = pkg_resources.resource_filename( 'circus', os.path.join('matlab_GUI', 'SortingGUI.m')) # Change to the directory of the matlab file os.chdir(os.path.abspath(os.path.dirname(gui_file))) # Use quotation marks for string arguments is_string = [False, True, True, True, False, True, False, False, True] arguments = ', '.join([ "'%s'" % arg if s else "%s" % arg for arg, s in zip(gui_params, is_string) ]) matlab_command = 'SortingGUI(%s)' % arguments print_and_log(["Launching the MATLAB GUI..."], 'info', params) try: sys.exit( subprocess.call( ['matlab', '-nodesktop', '-nosplash', '-r', matlab_command])) except Exception: print_error( ["Something wrong with MATLAB. Try circus-gui-python instead?"])
def main(argv=None): if argv is None: argv = sys.argv[1:] gheader = Fore.GREEN + get_header() header = gheader + Fore.RESET parser = argparse.ArgumentParser(description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file') parser.add_argument('-e', '--extension', help='extension to consider for visualization', default='') if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) filename = os.path.abspath(args.datafile) extension = args.extension params = circus.shared.utils.io.load_parameters(filename) mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12") if not mytest: print_and_log(['You need to update phy-contrib to the latest git version'], 'error', params) sys.exit(0) sampling_rate = float(params.getint('data', 'sampling_rate')) data_dtype = params.get('data', 'data_dtype') file_out_suff = params.get('data', 'file_out_suff') data_offset = params.getint('data', 'data_offset') probe = read_probe(params) if extension != '': extension = '-' + extension output_path = params.get('data', 'file_out_suff') + extension + '.GUI' if not os.path.exists(output_path): print_and_log(['Data should be first exported with the converting method!'], 'error', params) else: print_and_log(["Launching the phy GUI..."], 'info', params) gui_params = {} gui_params['dat_path'] = params.get('data', 'data_file') gui_params['n_channels_dat'] = params.getint('data', 'N_total') gui_params['n_features_per_channel'] = 5 gui_params['dtype'] = params.get('data', 'data_dtype') gui_params['offset'] = params.getint('data', 'data_offset') gui_params['sample_rate'] = params.getint('data', 'sampling_rate') gui_params['hp_filtered'] = True os.chdir(output_path) create_app() controller = TemplateController(**gui_params) gui = controller.create_gui() gui.show() run_app() gui.close() del gui
def main(argv=None): if argv is None: argv = sys.argv[1:] import h5py parallel_hdf5 = h5py.get_config().mpi from mpi4py import MPI try: SHARED_MEMORY = True MPI.Win.Allocate_shared(1, 1, MPI.INFO_NULL, MPI.COMM_SELF).Free() except (NotImplementedError, AttributeError): SHARED_MEMORY = False user_path = pjoin(os.path.expanduser('~'), 'spyking-circus') tasks_list = None if not os.path.exists(user_path): os.makedirs(user_path) try: import cudamat as cmt cmt.init() HAVE_CUDA = True except Exception: HAVE_CUDA = False all_steps = [ 'whitening', 'clustering', 'fitting', 'gathering', 'extracting', 'filtering', 'converting', 'benchmarking', 'merging', 'validating' ] if os.path.exists(user_path + 'config.params'): config_file = os.path.abspath(user_path + 'config.params') else: config_file = os.path.abspath( pkg_resources.resource_filename('circus', 'config.params')) gheader = Fore.GREEN + get_header() header = gheader header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str( psutil.cpu_count()) + '\n' header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str( HAVE_CUDA) + '\n' header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str( parallel_hdf5) + '\n' header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str( SHARED_MEMORY) + '\n' header += '\n' header += Fore.GREEN + "##################################################################" header += Fore.RESET method_help = '''by default, first 4 steps are performed, but a subset x,y can be done. Steps are: - filtering - whitening - clustering - fitting - (extra) merging [GUI for meta merging] - (extra) converting [export results to phy format] - (extra) gathering [force collection of results] - (extra) extracting [get templates from spike times] - (extra) benchmarking [with -o and -t] - (extra) validating [to compare performance with GT neurons]''' parser = argparse.ArgumentParser( description=header, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('datafile', help='data file (or a list of commands if batch mode)') parser.add_argument('-m', '--method', default='filtering,whitening,clustering,fitting', help=method_help) parser.add_argument('-c', '--cpu', type=int, default=1, help='number of CPU') parser.add_argument('-g', '--gpu', type=int, default=0, help='number of GPU') parser.add_argument('-H', '--hostfile', help='hostfile for MPI', default=pjoin(user_path, 'circus.hosts')) parser.add_argument( '-b', '--batch', help='datafile is a list of commands to launch, in a batch mode', action='store_true') parser.add_argument( '-p', '--preview', help='GUI to display the first second filtered with thresholds', action='store_true') parser.add_argument('-r', '--result', help='GUI to display the results on top of raw data', action='store_true') parser.add_argument( '-e', '--extension', help='extension to consider for merging and converting', default='None') parser.add_argument( '-o', '--output', help='output file [for generation of synthetic benchmarks]') parser.add_argument('-t', '--type', help='benchmark type', choices=['fitting', 'clustering', 'synchrony']) if len(argv) == 0: parser.print_help() sys.exit() args = parser.parse_args(argv) steps = args.method.split(',') for step in steps: if step not in all_steps: print_error(['The method "%s" is not recognized' % step]) sys.exit(1) # To save some typing later (nb_cpu, nb_gpu, hostfile, batch, preview, result, extension, output, benchmark) = (args.cpu, args.gpu, args.hostfile, args.batch, args.preview, args.result, args.extension, args.output, args.type) filename = os.path.abspath(args.datafile) f_next, extens = os.path.splitext(filename) if extens == '.params': print_error(['You should launch the code on the data file!']) sys.exit(1) file_params = f_next + '.params' if not os.path.exists(file_params) and not batch: print Fore.RED + 'The parameter file %s is not present!' % file_params key = '' while key not in ['y', 'n']: key = raw_input( Fore.WHITE + "Do you want SpyKING CIRCUS to create a parameter file? [y/n]") if key == 'y': print Fore.WHITE + "Generating template file", file_params print Fore.WHITE + "Fill it properly before launching the code! (see documentation)" shutil.copyfile(config_file, file_params) sys.exit() elif batch: tasks_list = filename if not batch: params = io.load_parameters(filename) if preview: print_info( ['Preview mode, showing only first second of the recording']) tmp_path_loc = os.path.join( os.path.abspath(params.get('data', 'data_file_noext')), 'tmp') if not os.path.exists(tmp_path_loc): os.makedirs(tmp_path_loc) filename = os.path.join(tmp_path_loc, os.path.basename(filename)) f_next, extens = os.path.splitext(filename) shutil.copyfile(file_params, f_next + '.params') steps = ['filtering', 'whitening'] io.prepare_preview(params, filename) io.change_flag(filename, 'chunk_size', '2') io.change_flag(filename, 'safety_time', '0') if tasks_list is not None: with open(tasks_list, 'r') as f: for line in f: if len(line) > 0: subprocess.check_call(['spyking-circus'] + line.replace('\n', '').split(" ")) else: if os.path.exists(f_next + '.log'): os.remove(f_next + '.log') write_to_logger(params, ['Config file: %s' % (f_next + '.params')], 'debug') write_to_logger(params, ['Data file : %s' % filename], 'debug') print gheader if preview: print Fore.GREEN + "Steps :", Fore.CYAN + "preview mode" elif result: print Fore.GREEN + "Steps :", Fore.CYAN + "results mode" else: print Fore.GREEN + "Steps :", Fore.CYAN + ", ".join(steps) print Fore.GREEN + "GPU detected :", Fore.CYAN + str(HAVE_CUDA) print Fore.GREEN + "Number of CPU :", Fore.CYAN + str( nb_cpu) + "/" + str(psutil.cpu_count()) if HAVE_CUDA: print Fore.GREEN + "Number of GPU :", Fore.CYAN + str(nb_gpu) print Fore.GREEN + "Parallel HDF5 :", Fore.CYAN + str(parallel_hdf5) print Fore.GREEN + "Shared memory :", Fore.CYAN + str(SHARED_MEMORY) print Fore.GREEN + "Hostfile :", Fore.CYAN + hostfile print "" print Fore.GREEN + "##################################################################" print "" print Fore.RESET # Launch the subtasks subtasks = [('filtering', 'mpirun'), ('whitening', 'mpirun'), ('clustering', 'mpirun'), ('fitting', 'mpirun'), ('extracting', 'mpirun'), ('gathering', 'python'), ('converting', 'mpirun'), ('benchmarking', 'mpirun'), ('merging', 'mpirun'), ('validating', 'mpirun')] if HAVE_CUDA and nb_gpu > 0: use_gpu = 'True' else: use_gpu = 'False' time = circus.shared.io.data_stats(params) / 60. if nb_cpu < psutil.cpu_count(): if use_gpu != 'True' and not result: io.print_and_log([ 'Using only %d out of %d local CPUs available (-c to change)' % (nb_cpu, psutil.cpu_count()) ], 'info', params) if params.getboolean('detection', 'matched-filter') and not params.getboolean( 'clustering', 'smart_search'): io.print_and_log( ['Smart Search should be activated for matched filtering'], 'info', params) if time > 30 and not params.getboolean('clustering', 'smart_search'): io.print_and_log( ['Smart Search could be activated for long recordings'], 'info', params) n_edges = circus.shared.io.get_averaged_n_edges(params) if n_edges > 100 and not params.getboolean('clustering', 'compress'): io.print_and_log([ 'Template compression is highly recommended based on parameters' ], 'info', params) if params.getint('data', 'N_e') > 500: if (params.getint('data', 'chunk_size') > 10) or (params.getint( 'whitening', 'chunk_size') > 10): io.print_and_log([ "Large number of electrodes, reduce chunk sizes to 10s in [data] and [whitening]" ], 'info', params) if not result: for subtask, command in subtasks: if subtask in steps: if command == 'python': # Directly call the launcher try: circus.launch(subtask, filename, nb_cpu, nb_gpu, use_gpu) except: print_error(['Step "%s" failed!' % subtask]) raise elif command == 'mpirun': # Use mpirun to make the call mpi_args = gather_mpi_arguments(hostfile, params) if subtask != 'fitting': nb_tasks = str(max(args.cpu, args.gpu)) else: if use_gpu == 'True': nb_tasks = str(args.gpu) else: nb_tasks = str(args.cpu) if subtask == 'benchmarking': if (output is None) or (benchmark is None): print_error([ "To generate synthetic datasets, you must provide output and type" ]) sys.exit() mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, output, benchmark ] elif subtask in ['merging', 'converting']: mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu, extension ] else: mpi_args += [ '-np', nb_tasks, 'spyking-circus-subtask', subtask, filename, str(nb_cpu), str(nb_gpu), use_gpu ] write_to_logger(params, ['Launching task %s' % subtask], 'debug') write_to_logger(params, ['Command: %s' % str(mpi_args)], 'debug') try: subprocess.check_call(mpi_args) except: print_error(['Step "%s" failed!' % subtask]) raise if preview or result: from circus.shared import gui import pylab from matplotlib.backends import qt_compat use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE if use_pyside: from PySide import QtGui, QtCore, uic else: from PyQt4 import QtGui, QtCore, uic app = QtGui.QApplication([]) try: pylab.style.use('ggplot') except Exception: pass if preview: mygui = gui.PreviewGUI(io.load_parameters(filename)) shutil.rmtree(tmp_path_loc) elif result: mygui = gui.PreviewGUI(io.load_parameters(filename), show_fit=True) sys.exit(app.exec_())
def main(filename, params, nb_cpu, nb_gpu, use_gpu, extension): params = circus.shared.utils.io.load_parameters(filename) sampling_rate = float(params.getint('data', 'sampling_rate')) data_dtype = params.get('data', 'data_dtype') file_out_suff = params.get('data', 'file_out_suff') data_offset = params.getint('data', 'data_offset') probe = read_probe(params) output_path = params.get('data', 'file_out_suff') + extension + '.GUI' N_e = params.getint('data', 'N_e') N_t = params.getint('data', 'N_t') erase_all = params.getboolean('converting', 'erase_all') export_pcs = params.get('converting', 'export_pcs') def generate_mapping(probe): p = {} positions = [] nodes = [] for key in probe['channel_groups'].keys(): p.update(probe['channel_groups'][key]['geometry']) nodes += probe['channel_groups'][key]['channels'] positions += [ p[channel] for channel in probe['channel_groups'][key]['channels'] ] idx = numpy.argsort(nodes) positions = numpy.array(positions)[idx] return positions def get_max_loc_channel(params): nodes, edges = circus.shared.utils.io.get_nodes_and_edges(params) max_loc_channel = 0 for key in edges.keys(): if len(edges[key]) > max_loc_channel: max_loc_channel = len(edges[key]) return max_loc_channel def write_results(path, params, extension): result = get_results(params, extension) spikes = numpy.zeros(0, dtype=numpy.uint64) clusters = numpy.zeros(0, dtype=numpy.uint32) amplitudes = numpy.zeros(0, dtype=numpy.double) for key in result['spiketimes'].keys(): temp_id = int(key.split('_')[-1]) data = result['spiketimes'].pop(key).astype(numpy.uint64) spikes = numpy.concatenate((spikes, data)) data = result['amplitudes'].pop(key).astype(numpy.double) amplitudes = numpy.concatenate((amplitudes, data[:, 0])) clusters = numpy.concatenate( (clusters, temp_id * numpy.ones(len(data), dtype=numpy.uint32))) idx = numpy.argsort(spikes) numpy.save(os.path.join(output_path, 'spike_templates'), clusters[idx]) numpy.save(os.path.join(output_path, 'spike_times'), spikes[idx]) numpy.save(os.path.join(output_path, 'amplitudes'), amplitudes[idx]) return def write_templates(path, params, extension): max_loc_channel = get_max_loc_channel(params) templates = load_data(params, 'templates', extension) N_tm = templates.shape[1] // 2 to_write = numpy.zeros((N_tm, N_t, N_e), dtype=numpy.float32) mapping = numpy.zeros((N_tm, max_loc_channel), dtype=numpy.int32) for t in xrange(N_tm): tmp = templates[:, t].toarray().reshape(N_e, N_t).T x, y = tmp.nonzero() to_write[t, x, y] = tmp[x, y] nb_loc = len(numpy.unique(y)) mapping[t, numpy.arange(nb_loc)] = numpy.unique(y) numpy.save(os.path.join(output_path, 'templates'), to_write.astype(numpy.single)) numpy.save(os.path.join(output_path, 'templates_ind'), mapping.astype(numpy.double)) return N_tm def write_pcs(path, params, comm, extension, mode=0): spikes = numpy.load(os.path.join(output_path, 'spike_times.npy')) labels = numpy.load(os.path.join(output_path, 'spike_templates.npy')) max_loc_channel = get_max_loc_channel(params) nb_features = params.getint('whitening', 'output_dim') nodes, edges = get_nodes_and_edges(params) N_total = params.getint('data', 'N_total') templates = load_data(params, 'templates', extension) N_tm = templates.shape[1] // 2 pc_features_ind = numpy.zeros((N_tm, max_loc_channel), dtype=numpy.int32) clusters = load_data(params, 'clusters', extension) best_elec = clusters['electrodes'] inv_nodes = numpy.zeros(N_total, dtype=numpy.int32) inv_nodes[nodes] = numpy.argsort(nodes) for count, elec in enumerate(best_elec): nb_loc = len(edges[nodes[elec]]) pc_features_ind[count, numpy.arange(nb_loc)] = inv_nodes[edges[ nodes[elec]]] basis_proj, basis_rec = load_data(params, 'basis') to_process = numpy.arange(comm.rank, N_tm, comm.size) all_offsets = numpy.zeros(N_tm, dtype=numpy.int32) for target in xrange(N_tm): if mode == 0: all_offsets[target] = len(numpy.where(labels == target)[0]) elif mode == 1: all_offsets[target] = min( 500, len(numpy.where(labels == target)[0])) all_paddings = numpy.concatenate(([0], numpy.cumsum(all_offsets))) total_pcs = numpy.sum(all_offsets) pc_file = os.path.join(output_path, 'pc_features.npy') pc_file_ids = os.path.join(output_path, 'pc_feature_spike_ids.npy') from numpy.lib.format import open_memmap if comm.rank == 0: pc_features = open_memmap(pc_file, shape=(total_pcs, nb_features, max_loc_channel), dtype=numpy.float32, mode='w+') if mode == 1: pc_ids = open_memmap(pc_file_ids, shape=(total_pcs, ), dtype=numpy.int32, mode='w+') comm.Barrier() pc_features = open_memmap(pc_file, mode='r+') if mode == 1: pc_ids = open_memmap(pc_file_ids, mode='r+') if comm.rank == 0: pbar = get_progressbar(len(to_process)) all_idx = numpy.zeros(0, dtype=numpy.int32) for gcount, target in enumerate(to_process): count = all_paddings[target] if mode == 1: idx = numpy.random.permutation( numpy.where(labels == target)[0])[:500] pc_ids[count:count + len(idx)] = idx elif mode == 0: idx = numpy.where(labels == target)[0] elec = best_elec[target] indices = inv_nodes[edges[nodes[elec]]] labels_i = target * numpy.ones(len(idx)) times_i = numpy.take(spikes, idx) sub_data = get_stas(params, times_i, labels_i, elec, neighs=indices, nodes=nodes, auto_align=False) pcs = numpy.dot(sub_data, basis_proj) pcs = numpy.swapaxes(pcs, 1, 2) if mode == 0: pc_features[idx, :, :len(indices)] = pcs elif mode == 1: pc_features[count:count + len(idx), :, :len(indices)] = pcs if comm.rank == 0: pbar.update(gcount) if comm.rank == 0: pbar.finish() comm.Barrier() if comm.rank == 0: numpy.save(os.path.join(output_path, 'pc_feature_ind'), pc_features_ind.astype( numpy.uint32)) #n_templates, n_loc_chan do_export = True if comm.rank == 0: if os.path.exists(output_path): if not erase_all: key = '' while key not in ['y', 'n']: print( Fore.WHITE + "Export already made! Do you want to erase everything? (y)es / (n)o " ) key = raw_input('') if key == 'y': do_export = True else: do_export = False if do_export: if os.path.exists(os.path.abspath('.phy')): shutil.rmtree(os.path.abspath('.phy')) shutil.rmtree(output_path) if do_export == True: comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) elif do_export == False: comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) else: do_export = bool( comm.bcast(numpy.array([0], dtype=numpy.int32), root=0)) comm.Barrier() if do_export: if comm.rank == 0: os.makedirs(output_path) print_and_log( ["Exporting data for the phy GUI with %d CPUs..." % nb_cpu], 'info', params) numpy.save( os.path.join(output_path, 'whitening_mat'), load_data(params, 'spatial_whitening').astype(numpy.double)) numpy.save(os.path.join(output_path, 'channel_positions'), generate_mapping(probe).astype(numpy.double)) nodes, edges = get_nodes_and_edges(params) numpy.save(os.path.join(output_path, 'channel_map'), nodes.astype(numpy.int32)) write_results(output_path, params, extension) N_tm = write_templates(output_path, params, extension) similarities = h5py.File(file_out_suff + '.templates%s.hdf5' % extension, 'r+', libver='latest').get('maxoverlap') norm = params.getint('data', 'N_e') * params.getint('data', 'N_t') numpy.save(os.path.join(output_path, 'similar_templates'), (similarities[:N_tm, :N_tm] / norm).astype( numpy.single)) comm.Barrier() make_pcs = 2 if comm.rank == 0: if export_pcs == 'prompt': key = '' while key not in ['a', 's', 'n']: print( Fore.WHITE + "Do you want SpyKING CIRCUS to export PCs? (a)ll / (s)ome / (n)o" ) key = raw_input('') else: key = export_pcs if key == 'a': make_pcs = 0 comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) elif key == 's': make_pcs = 1 comm.bcast(numpy.array([1], dtype=numpy.int32), root=0) elif key == 'n': comm.bcast(numpy.array([2], dtype=numpy.int32), root=0) if os.path.exists(os.path.join(output_path, 'pc_features.npy')): os.remove(os.path.join(output_path, 'pc_features.npy')) if os.path.exists( os.path.join(output_path, 'pc_feature_ind.npy')): os.remove(os.path.join(output_path, 'pc_feature_ind.npy')) else: make_pcs = comm.bcast(numpy.array([0], dtype=numpy.int32), root=0) make_pcs = make_pcs[0] comm.Barrier() if make_pcs < 2: write_pcs(output_path, params, comm, extension, make_pcs)