import numpy as np
import os
from circus.shared.parser import CircusParser
# Parse arguments.
parser = argparse.ArgumentParser(description="Inspect basis waveforms.")
parser.add_argument('datafile', help="data file")
args = parser.parse_args()
# Load parameters.
params = CircusParser(args.datafile)
_ = params.get_data_file()
sampling_rate = params.rate
file_out_suff = params.get('data', 'file_out_suff')
nb_channels = params.getint('data', 'N_e')
nb_time_steps = params.getint('detection', 'N_t')
# Load basis waveforms.
basis_path = "{}.basis.hdf5".format(file_out_suff)
if not os.path.isfile(basis_path):
raise FileNotFoundError(basis_path)
with h5py.File(basis_path, mode='r', libver='earliest') as basis_file:
if 'proj' not in basis_file:
raise RuntimeError(
"No projection matrix found in {}.".format(basis_path))
projection_matrix = basis_file['proj'][:]
assert projection_matrix.shape[0] == nb_time_steps, (
projection_matrix.shape, nb_time_steps)
_, nb_components = projection_matrix.shape
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
os.environ['OPENBLAS_NUM_THREADS'] = str(1)
parallel_hdf5 = h5py.get_config().mpi
user_path = pjoin(os.path.expanduser('~'), 'spyking-circus')
tasks_list = None
if not os.path.exists(user_path):
os.makedirs(user_path)
try:
import cudamat as cmt
cmt.init()
HAVE_CUDA = True
except Exception:
HAVE_CUDA = False
all_steps = [
'whitening', 'clustering', 'fitting', 'gathering', 'extracting',
'filtering', 'converting', 'benchmarking', 'merging', 'validating'
]
if os.path.exists(pjoin(user_path, 'config.params')):
config_file = os.path.abspath(pjoin(user_path, 'config.params'))
else:
config_file = os.path.abspath(
pkg_resources.resource_filename('circus', 'config.params'))
header = get_colored_header()
header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str(
psutil.cpu_count()) + '\n'
header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str(
HAVE_CUDA) + '\n'
header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str(
parallel_hdf5) + '\n'
header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str(
SHARED_MEMORY) + '\n'
header += '\n'
header += Fore.GREEN + "##################################################################"
header += Fore.RESET
method_help = '''by default, first 4 steps are performed,
but a subset x,y can be done. Steps are:
- filtering
- whitening
- clustering
- fitting
- (extra) merging [GUI for meta merging]
- (extra) converting [export results to phy format]
- (extra) gathering [force collection of results]
- (extra) extracting [get templates from spike times]
- (extra) benchmarking [with -o and -t]
- (extra) validating [to compare performance with GT neurons]'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile',
help='data file (or a list of commands if batch mode)')
parser.add_argument(
'-i',
'--info',
help='list the file formats supported by SpyKING CIRCUS',
action='store_true')
parser.add_argument('-m',
'--method',
default='filtering,whitening,clustering,fitting',
help=method_help)
parser.add_argument('-c',
'--cpu',
type=int,
default=1,
help='number of CPU')
parser.add_argument('-g',
'--gpu',
type=int,
default=0,
help='number of GPU')
parser.add_argument('-H',
'--hostfile',
help='hostfile for MPI',
default=pjoin(user_path, 'circus.hosts'))
parser.add_argument(
'-b',
'--batch',
help='datafile is a list of commands to launch, in a batch mode',
action='store_true')
parser.add_argument(
'-p',
'--preview',
help='GUI to display the first second filtered with thresholds',
action='store_true')
parser.add_argument('-r',
'--result',
help='GUI to display the results on top of raw data',
action='store_true')
parser.add_argument('-s',
'--second',
type=int,
default=0,
help='If preview mode, begining of the preview [in s]')
parser.add_argument(
'-e',
'--extension',
help='extension to consider for merging and converting',
default='None')
parser.add_argument(
'-o',
'--output',
help='output file [for generation of synthetic benchmarks]')
parser.add_argument('-t',
'--type',
help='benchmark type',
choices=['fitting', 'clustering', 'synchrony'])
if len(argv) == 0:
parser.print_help()
sys.exit(0)
args = parser.parse_args(argv)
steps = args.method.split(',')
for step in steps:
if step not in all_steps:
print_error(['The method "%s" is not recognized' % step])
sys.exit(0)
# To save some typing later
(nb_cpu, nb_gpu, hostfile, batch, preview, result, extension, output,
benchmark, info,
second) = (args.cpu, args.gpu, args.hostfile, args.batch, args.preview,
args.result, args.extension, args.output, args.type, args.info,
args.second)
filename = os.path.abspath(args.datafile)
f_next, extens = os.path.splitext(filename)
if info:
if args.datafile.lower() in __supported_data_files__:
filename = 'tmp'
if len(__supported_data_files__[
args.datafile.lower()].extension) > 0:
filename += __supported_data_files__[
args.datafile.lower()].extension[0]
__supported_data_files__[args.datafile.lower()](
filename, {}, is_empty=True)._display_requirements_()
else:
print_and_log([
'', 'To get info on any particular file format, do:',
'>> spyking-circus file_format -i', ''
], 'default')
print_and_log(list_all_file_format())
sys.exit(0)
if extens == '.params':
print_error(['You should launch the code on the data file!'])
sys.exit(0)
file_params = f_next + '.params'
if not os.path.exists(file_params) and not batch:
print Fore.RED + 'The parameter file %s is not present!' % file_params
create_params = query_yes_no(
Fore.WHITE +
"Do you want SpyKING CIRCUS to create a parameter file?")
if create_params:
print Fore.WHITE + "Creating", file_params
print Fore.WHITE + "Fill it properly before launching the code! (see documentation)"
print_info([
'Keep in mind that filtering is performed on site, so please',
'be sure to keep a copy of your data elsewhere'
])
shutil.copyfile(config_file, file_params)
sys.exit(0)
elif batch:
tasks_list = filename
if not batch:
logfile = f_next + '.log'
if os.path.exists(logfile):
os.remove(logfile)
logger = init_logging(logfile)
params = CircusParser(filename)
data_file = params.get_data_file(source=True, has_been_created=False)
overwrite = params.getboolean('data', 'overwrite')
file_format = params.get('data', 'file_format')
if overwrite:
support_parallel_write = data_file.parallel_write
is_writable = data_file.is_writable
else:
support_parallel_write = __supported_data_files__[
'raw_binary'].parallel_write
is_writable = __supported_data_files__['raw_binary'].is_writable
if preview:
print_and_log([
'Preview mode, showing only seconds [%d-%d] of the recording' %
(second, second + 2)
], 'info', logger)
tmp_path_loc = os.path.join(
os.path.abspath(params.get('data', 'data_file_noext')), 'tmp')
if not os.path.exists(tmp_path_loc):
os.makedirs(tmp_path_loc)
filename = os.path.join(tmp_path_loc, os.path.basename(filename))
f_next, extens = os.path.splitext(filename)
shutil.copyfile(file_params, f_next + '.params')
steps = ['filtering', 'whitening']
chunk_size = int(2 * params.rate)
data_file.open()
nb_chunks, _ = data_file.analyze(chunk_size)
if nb_chunks < (second + 2):
print_and_log([
'Recording is too short to display seconds [%d-%d]' %
(second, second + 2)
])
sys.exit(0)
local_chunk, t_offset = data_file.get_data(second, second + chunk_size)
description = data_file.get_description()
data_file.close()
new_params = CircusParser(filename)
new_params.write('data', 'chunk_size', '2')
new_params.write('data', 'file_format', 'raw_binary')
new_params.write('data', 'data_dtype', 'float32')
new_params.write('data', 'data_offset', '0')
new_params.write('data', 'dtype_offset', '0')
new_params.write('data', 'stream_mode', 'None')
new_params.write('data', 'overwrite', 'True')
new_params.write('data', 'sampling_rate', str(params.rate))
new_params.write('whitening', 'safety_time', '0')
new_params.write('clustering', 'safety_time', '0')
new_params.write('whitening', 'chunk_size', '2')
preview_params = os.path.abspath(params.get(
'data', 'data_file_noext')) + '.params'
new_params.write('data', 'preview_path', preview_params)
description['data_dtype'] = 'float32'
description['dtype_offset'] = 0
description['data_offset'] = 0
description['gain'] = 1.
new_params = CircusParser(filename)
data_file_out = new_params.get_data_file(is_empty=True,
params=description)
support_parallel_write = data_file_out.parallel_write
is_writable = data_file_out.is_writable
data_file_out.allocate(shape=local_chunk.shape,
data_dtype=numpy.float32)
data_file_out.open('r+')
data_file_out.set_data(0, local_chunk)
data_file_out.close()
if tasks_list is not None:
with open(tasks_list, 'r') as f:
for line in f:
if len(line) > 0:
subprocess.check_call(['spyking-circus'] +
line.replace('\n', '').split(" "))
else:
print_and_log(['Config file: %s' % (f_next + '.params')], 'debug',
logger)
print_and_log(['Data file : %s' % filename], 'debug', logger)
print get_colored_header()
if preview:
print Fore.GREEN + "Steps :", Fore.CYAN + "preview mode"
elif result:
print Fore.GREEN + "Steps :", Fore.CYAN + "result mode"
else:
print Fore.GREEN + "Steps :", Fore.CYAN + ", ".join(steps)
print Fore.GREEN + "GPU detected :", Fore.CYAN + str(HAVE_CUDA)
print Fore.GREEN + "Number of CPU :", Fore.CYAN + str(
nb_cpu) + "/" + str(psutil.cpu_count())
if HAVE_CUDA:
print Fore.GREEN + "Number of GPU :", Fore.CYAN + str(nb_gpu)
print Fore.GREEN + "Parallel HDF5 :", Fore.CYAN + str(parallel_hdf5)
do_upgrade = ''
if not SHARED_MEMORY:
do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]'
print Fore.GREEN + "Shared memory :", Fore.CYAN + str(
SHARED_MEMORY) + do_upgrade
print Fore.GREEN + "Hostfile :", Fore.CYAN + hostfile
print ""
print Fore.GREEN + "##################################################################"
print ""
print Fore.RESET
# Launch the subtasks
subtasks = [('filtering', 'mpirun'), ('whitening', 'mpirun'),
('clustering', 'mpirun'), ('fitting', 'mpirun'),
('extracting', 'mpirun'), ('gathering', 'python'),
('converting', 'mpirun'), ('benchmarking', 'mpirun'),
('merging', 'mpirun'), ('validating', 'mpirun')]
if HAVE_CUDA and nb_gpu > 0:
use_gpu = 'True'
else:
use_gpu = 'False'
time = data_stats(params) / 60.
if preview:
params = new_params
if nb_cpu < psutil.cpu_count():
if use_gpu != 'True' and not result:
print_and_log([
'Using only %d out of %d local CPUs available (-c to change)'
% (nb_cpu, psutil.cpu_count())
], 'info', logger)
if params.getboolean('detection',
'matched-filter') and not params.getboolean(
'clustering', 'smart_search'):
print_and_log(
['Smart Search should be activated for matched filtering'],
'info', logger)
if time > 30 and not params.getboolean('clustering', 'smart_search'):
print_and_log(
['Smart Search could be activated for long recordings'],
'info', logger)
n_edges = get_averaged_n_edges(params)
if n_edges > 100 and not params.getboolean('clustering', 'compress'):
print_and_log([
'Template compression is highly recommended based on parameters'
], 'info', logger)
if params.getint('data', 'N_e') > 500:
if params.getint('data', 'chunk_size') > 10:
params.write('data', 'chunk_size', '10')
if params.getint('whitening', 'chunk_size') > 10:
params.write('whitening', 'chunk_size', '10')
print_and_log(
["Large number of electrodes, reducing chunk sizes to 10s"],
'info', logger)
if not result:
for subtask, command in subtasks:
if subtask in steps:
if command == 'python':
# Directly call the launcher
try:
circus.launch(subtask, filename, nb_cpu, nb_gpu,
use_gpu)
except:
print_and_log(['Step "%s" failed!' % subtask],
'error', logger)
sys.exit(0)
elif command == 'mpirun':
# Use mpirun to make the call
mpi_args = gather_mpi_arguments(hostfile, params)
if subtask in ['filtering', 'benchmarking'
] and not is_writable:
if not preview and overwrite:
print_and_log([
'The file format %s is read only!' %
file_format,
'You should set overwite to False, to create a copy of the data.',
'However, note that if you have streams, informations on times',
'will be discarded'
], 'info', logger)
sys.exit(0)
if subtask in [
'filtering'
] and not support_parallel_write and (args.cpu > 1):
print_and_log([
'No parallel writes for %s: only 1 node used for %s'
% (file_format, subtask)
], 'info', logger)
nb_tasks = str(1)
else:
if subtask != 'fitting':
nb_tasks = str(max(args.cpu, args.gpu))
else:
if use_gpu == 'True':
nb_tasks = str(args.gpu)
else:
nb_tasks = str(args.cpu)
if subtask == 'benchmarking':
if (output is None) or (benchmark is None):
print_and_log([
"To generate synthetic datasets, you must provide output and type"
], 'error', logger)
sys.exit(0)
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu, output, benchmark
]
elif subtask in ['merging', 'converting']:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu, extension
]
else:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename,
str(nb_cpu),
str(nb_gpu), use_gpu
]
print_and_log(['Launching task %s' % subtask], 'debug',
logger)
print_and_log(['Command: %s' % str(mpi_args)], 'debug',
logger)
try:
subprocess.check_call(mpi_args)
except:
print_and_log(['Step "%s" failed!' % subtask],
'error', logger)
raise
if preview or result:
from circus.shared import gui
import pylab
from matplotlib.backends import qt_compat
use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE
if use_pyside:
from PySide import QtGui, QtCore, uic
else:
from PyQt4 import QtGui, QtCore, uic
app = QtGui.QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
if preview:
print_and_log(['Launching the preview GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(new_params)
shutil.rmtree(tmp_path_loc)
elif result:
data_file = params.get_data_file()
print_and_log(['Launching the result GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(params, show_fit=True)
sys.exit(app.exec_())
import matplotlib.pyplot as plt
import numpy as np
import os
from circus.shared.parser import CircusParser
# Parse arguments.
parser = argparse.ArgumentParser(description="Inspect threhsolds.")
parser.add_argument('datafile', help="data file")
args = parser.parse_args()
# Load parameters.
params = CircusParser(args.datafile)
_ = params.get_data_file()
file_out_suff = params.get('data', 'file_out_suff')
nb_channels = params.getint('data', 'N_e')
# Load spatial matrix.
basis_path = "{}.basis.hdf5".format(file_out_suff)
if not os.path.isfile(basis_path):
raise FileNotFoundError(basis_path)
with h5py.File(basis_path, mode='r', libver='earliest') as basis_file:
if 'thresholds' not in basis_file:
raise RuntimeError("No thresholds found in {}".format(basis_path))
thresholds = basis_file['thresholds'][:]
assert thresholds.shape == (nb_channels, ), (thresholds.shape, nb_channels)
# Plot thresholds.
fig, axes = plt.subplots(nrows=2, squeeze=False, sharex=True)
# # 1st subplot.
ax = axes[0, 0]
'--color',
choices=['amplitude', 'norm'],
help="marker color")
parser.add_argument('-n',
'--nb-snippets-max',
default=20,
type=int,
help="maximum number of snippets")
args = parser.parse_args()
# Load parameters.
params = CircusParser(args.datafile)
_ = params.get_data_file()
sampling_rate = params.rate
# duration = params.data_file.duration
nb_time_steps = params.getint('detection', 'N_t')
# Load spike times.
results = load_data(params, 'results',
extension='') # TODO support other extensions?
results_data = dict()
for key in results['spiketimes'].keys():
if key[:5] == 'temp_':
template_key = key
template_id = int(key[5:])
if 'spike_times' not in results_data:
results_data['spike_times'] = dict()
results_data['spike_times'][template_id] = results['spiketimes'][
template_key][:]
# Load peak times.
parser.add_argument('datafile', help="data file")
parser.add_argument('-t',
'--template',
default=0,
type=int,
help="template index",
dest='template_id')
args = parser.parse_args()
# Load parameters.
params = CircusParser(args.datafile)
_ = params.get_data_file()
nb_electrodes = params.nb_channels
sampling_rate = params.rate # Hz
duration = params.data_file.duration / sampling_rate # s
nb_channels = params.getint('data', 'N_e')
nb_time_steps = params.getint('detection', 'N_t')
nb_snippets = params.getint('clustering', 'nb_snippets')
# Load snippets.
clusters_data = load_clusters_data(params, extension='')
electrode = clusters_data['electrodes'][args.template_id]
local_cluster = clusters_data['local_clusters'][args.template_id]
assert electrode.shape == local_cluster.shape, (electrode.shape,
local_cluster.shape)
times = clusters_data['times'][electrode]
clusters = clusters_data['clusters'][electrode]
assert times.shape == clusters.shape, (times.shape, clusters.shape)
selection = (clusters == local_cluster)
times = times[selection]
clusters = clusters[selection]
