def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to split results obtained when using N streams
into N individual results files, one per stream
'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for slicing results',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
if extension != '':
extension = '-' + extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
_ = init_logging(params.logfile)
logger = logging.getLogger(__name__)
file_out_suff = params.get('data', 'file_out_suff')
if params.get('data', 'stream_mode') in ['None', 'none']:
print_and_log(['No streams in the datafile!'], 'error', logger)
sys.exit(1)
data_file = params.get_data_file()
result = circus.shared.files.get_results(params, extension=extension)
times = []
for source in data_file._sources:
times += [[source.t_start, source.t_stop]]
sub_results = slice_result(result, times)
for count, result in enumerate(sub_results):
keys = ['spiketimes', 'amplitudes']
mydata = h5py.File(file_out_suff + '.result%s_%d.hdf5' %
(extension, count),
'w',
libver='earliest')
for key in keys:
mydata.create_group(key)
for temp in result[key].keys():
tmp_path = '%s/%s' % (key, temp)
mydata.create_dataset(tmp_path, data=result[key][temp])
mydata.close()
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to launch the phy GUI and visualize the results.
[data must be first converted with the converting mode]
'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
if extension != '':
extension = '-' + extension
try:
import traitlets
except ImportError:
print_and_log(
['The package traitlets required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import click
except ImportError:
print_and_log(['The package click required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import joblib
except ImportError:
print_and_log(['The package joblib required by phy is not installed'],
'error', logger)
sys.exit(1)
if HAVE_PHYCONTRIB:
mytest = StrictVersion(
phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(
['You need to update phy-contrib to the latest git version'],
'error', logger)
sys.exit(1)
print_and_log([
'phy-contrib is deprecated, you should upgrade to phy 2.0 and phylib'
], 'info', logger)
if HAVE_PHYLIB:
try:
import colorcet
except ImportError:
print_and_log(
['The package colorcet required by phy is not installed'],
'error', logger)
sys.exit(1)
try:
import qtconsole
except ImportError:
print_and_log(
['The package qtconsole required by phy is not installed'],
'error', logger)
sys.exit(1)
if not test_patch_for_similarities(params, extension):
print_and_log(
['You should re-export the data because of a fix in 0.6'], 'error',
logger)
continue_anyway = query_yes_no(
Fore.WHITE + "Continue anyway (results may not be fully correct)?",
default=None)
if not continue_anyway:
sys.exit(1)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
if data_file.params.has_key('data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
file_format = data_file.description
file_out_suff = params.get('data', 'file_out_suff')
if file_format not in supported_by_phy:
print_and_log([
"File format %s is not supported by phy. TraceView disabled" %
file_format
], 'info', logger)
if numpy.iterable(data_file.gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
else:
if data_file.gain != 1:
print_and_log([
"Gain of %g is not supported by phy. Expecting a scaling mismatch"
% data_file.gain
], 'info', logger)
gain = data_file.gain
probe = params.probe
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(
['Data should be first exported with the converting method!'],
'error', logger)
else:
print_and_log(["Launching the phy GUI..."], 'info', logger)
gui_params = {}
if file_format in supported_by_phy:
if not params.getboolean('data', 'overwrite'):
gui_params['dat_path'] = r"%s" % params.get(
'data', 'data_file_no_overwrite')
else:
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True,
has_been_created=False)
gui_params['dat_path'] = [
r"%s" % f for f in data_file.get_file_names()
]
else:
gui_params['dat_path'] = r"%s" % params.get(
'data', 'data_file')
else:
gui_params['dat_path'] = 'giverandomname.dat'
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_dtype
gui_params['offset'] = data_offset
gui_params['sample_rate'] = params.rate
gui_params['dir_path'] = output_path
gui_params['hp_filtered'] = True
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(
['You need to update phy-contrib to the latest git version'],
'error', logger)
sys.exit(1)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
if hasattr(data_file, 'data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
file_format = data_file.description
file_out_suff = params.get('data', 'file_out_suff')
if file_format not in supported_by_phy:
print_and_log([
"File format %s is not supported by phy. TraceView disabled" %
file_format
], 'info', logger)
if numpy.iterable(data_file.gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
else:
if data_file.gain != 1:
print_and_log([
"Gain of %g is not supported by phy. Expecting a scaling mismatch"
% gain
], 'info', logger)
gain = data_file.gain
probe = params.probe
if extension != '':
extension = '-' + extension
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(
['Data should be first exported with the converting method!'],
'error', logger)
else:
print_and_log(["Launching the phy GUI..."], 'info', logger)
gui_params = {}
if file_format in supported_by_phy:
gui_params['dat_path'] = params.get('data', 'data_file')
else:
gui_params['dat_path'] = ''
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_dtype
gui_params['offset'] = data_offset
gui_params['sample_rate'] = params.rate
gui_params['hp_filtered'] = True
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
gain = data_file.gain
t_start = data_file.t_start
file_format = data_file.description
if file_format not in supported_by_matlab:
print_and_log([
"File format %s is not supported by MATLAB. Waveforms disabled" %
file_format
], 'info', logger)
if numpy.iterable(gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
file_out_suff = params.get('data', 'file_out_suff')
if hasattr(data_file, 'data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
probe = params.probe
if extension != '':
extension = '-' + extension
def generate_matlab_mapping(probe):
p = {}
positions = []
nodes = []
for key in probe['channel_groups'].keys():
p.update(probe['channel_groups'][key]['geometry'])
nodes += probe['channel_groups'][key]['channels']
positions += [
p[channel]
for channel in probe['channel_groups'][key]['channels']
]
idx = numpy.argsort(nodes)
positions = numpy.array(positions)[idx]
t = tempfile.NamedTemporaryFile().name + '.hdf5'
cfile = h5py.File(t, 'w')
to_write = {
'positions': positions / 10.,
'permutation': numpy.sort(nodes),
'nb_total': numpy.array([probe['total_nb_channels']])
}
write_datasets(cfile, to_write.keys(), to_write)
cfile.close()
return t
mapping = generate_matlab_mapping(probe)
if not params.getboolean('data', 'overwrite'):
filename = params.get('data', 'data_file_no_overwrite')
else:
filename = params.get('data', 'data_file')
apply_patch_for_similarities(params, extension)
gui_file = pkg_resources.resource_filename(
'circus', os.path.join('matlab_GUI', 'SortingGUI.m'))
# Change to the directory of the matlab file
os.chdir(os.path.abspath(os.path.dirname(gui_file)))
# Use quotation marks for string arguments
if file_format not in supported_by_matlab:
gui_params = [
params.rate,
os.path.abspath(file_out_suff),
'%s.mat' % extension, mapping, 2, t_start
]
is_string = [False, True, True, True, False]
else:
gui_params = [
params.rate,
os.path.abspath(file_out_suff),
'%s.mat' % extension, mapping, 2, t_start, data_dtype, data_offset,
gain, filename
]
is_string = [
False, True, True, True, False, False, True, False, False, True
]
arguments = ', '.join([
"'%s'" % arg if s else "%s" % arg
for arg, s in zip(gui_params, is_string)
])
matlab_command = 'SortingGUI(%s)' % arguments
print_and_log(["Launching the MATLAB GUI..."], 'info', logger)
print_and_log([matlab_command], 'debug', logger)
if params.getboolean('fitting', 'collect_all'):
print_and_log([
'You can not view the unfitted spikes with the MATLAB GUI',
'Please consider using phy if you really would like to see them'
], 'info', logger)
try:
sys.exit(
subprocess.call(
['matlab', '-nodesktop', '-nosplash', '-r', matlab_command]))
except Exception:
print_and_log(
["Something wrong with MATLAB. Try circus-gui-python instead?"],
'error', logger)
sys.exit(1)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
parser.add_argument('-e',
'--extension',
help='extension to consider for visualization',
default='')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
filename = os.path.abspath(args.datafile)
extension = args.extension
params = CircusParser(filename)
if os.path.exists(params.logfile):
os.remove(params.logfile)
logger = init_logging(params.logfile)
logger = logging.getLogger(__name__)
mytest = StrictVersion(phycontrib.__version__) >= StrictVersion("1.0.12")
if not mytest:
print_and_log(
['You need to update phy-contrib to the latest git version'],
'error', logger)
sys.exit(1)
if not test_patch_for_similarities(params, extension):
print_and_log(
['You should re-export the data because of a fix in 0.6'], 'error',
logger)
continue_anyway = query_yes_no(
Fore.WHITE + "Continue anyway (results may not be fully correct)?",
default=None)
if not continue_anyway:
sys.exit(1)
data_file = params.get_data_file()
data_dtype = data_file.data_dtype
if data_file.params.has_key('data_offset'):
data_offset = data_file.data_offset
else:
data_offset = 0
file_format = data_file.description
file_out_suff = params.get('data', 'file_out_suff')
if file_format not in supported_by_phy:
print_and_log([
"File format %s is not supported by phy. TraceView disabled" %
file_format
], 'info', logger)
if numpy.iterable(data_file.gain):
print_and_log(
['Multiple gains are not supported, using a default value of 1'],
'info', logger)
gain = 1
else:
if data_file.gain != 1:
print_and_log([
"Gain of %g is not supported by phy. Expecting a scaling mismatch"
% data_file.gain
], 'info', logger)
gain = data_file.gain
probe = params.probe
if extension != '':
extension = '-' + extension
output_path = params.get('data', 'file_out_suff') + extension + '.GUI'
if not os.path.exists(output_path):
print_and_log(
['Data should be first exported with the converting method!'],
'error', logger)
else:
print_and_log(["Launching the phy GUI..."], 'info', logger)
gui_params = {}
if file_format in supported_by_phy:
if not params.getboolean('data', 'overwrite'):
gui_params['dat_path'] = params.get('data',
'data_file_no_overwrite')
else:
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True,
has_been_created=False)
gui_params['dat_path'] = ' '.join(
data_file.get_file_names())
else:
gui_params['dat_path'] = params.get('data', 'data_file')
else:
gui_params['dat_path'] = 'giverandomname.dat'
gui_params['n_channels_dat'] = params.nb_channels
gui_params['n_features_per_channel'] = 5
gui_params['dtype'] = data_dtype
gui_params['offset'] = data_offset
gui_params['sample_rate'] = params.rate
gui_params['dir_path'] = output_path
gui_params['hp_filtered'] = True
f = open(os.path.join(output_path, 'params.py'), 'w')
for key, value in gui_params.items():
if key in ['dir_path', 'dat_path', 'dtype']:
f.write('%s = "%s"\n' % (key, value))
else:
f.write("%s = %s\n" % (key, value))
f.close()
os.chdir(output_path)
create_app()
controller = TemplateController(**gui_params)
gui = controller.create_gui()
gui.show()
run_app()
gui.close()
del gui
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parallel_hdf5 = h5py.get_config().mpi
user_path = pjoin(os.path.expanduser('~'), 'spyking-circus')
tasks_list = None
if not os.path.exists(user_path):
os.makedirs(user_path)
try:
import cudamat as cmt
cmt.init()
HAVE_CUDA = True
except Exception:
HAVE_CUDA = False
all_steps = [
'whitening', 'clustering', 'fitting', 'gathering', 'extracting',
'filtering', 'converting', 'deconverting', 'benchmarking',
'merging', 'validating', 'thresholding'
]
config_file = os.path.abspath(pkg_resources.resource_filename('circus', 'config.params'))
header = get_colored_header()
header += Fore.GREEN + 'Local CPUs : ' + Fore.CYAN + str(psutil.cpu_count()) + '\n'
# header += Fore.GREEN + 'GPU detected : ' + Fore.CYAN + str(HAVE_CUDA) + '\n'
header += Fore.GREEN + 'Parallel HDF5 : ' + Fore.CYAN + str(parallel_hdf5) + '\n'
do_upgrade = ''
if not SHARED_MEMORY:
do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]'
header += Fore.GREEN + 'Shared memory : ' + Fore.CYAN + str(SHARED_MEMORY) + do_upgrade + '\n'
header += '\n'
header += Fore.GREEN + "##################################################################"
header += Fore.RESET
method_help = '''by default, all steps are performed,
but a subset x,y can be done. Steps are:
- filtering
- whitening
- clustering
- fitting
- merging [with or without a GUI for meta merging]
- (extra) converting [export results to phy format]
- (extra) thresholding [to get MUA activity only]
- (extra) deconverting [import results from phy format]
- (extra) gathering [force collection of results]
- (extra) extracting [get templates from spike times]
- (extra) benchmarking [with -o and -t]
- (extra) validating [to compare performance with GT neurons]'''
parser = argparse.ArgumentParser(description=header,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file (or a list of commands if batch mode)')
parser.add_argument('-i', '--info', help='list the file formats supported by SpyKING CIRCUS', action='store_true')
parser.add_argument('-m', '--method',
default='filtering,whitening,clustering,fitting,merging',
help=method_help)
parser.add_argument('-c', '--cpu', type=int, default=max(1, int(psutil.cpu_count()/2)), help='number of CPU')
# parser.add_argument('-g', '--gpu', type=int, default=0, help='number of GPU')
parser.add_argument('-H', '--hostfile', help='hostfile for MPI',
default=pjoin(user_path, 'circus.hosts'))
parser.add_argument('-b', '--batch', help='datafile is a list of commands to launch, in a batch mode',
action='store_true')
parser.add_argument('-p', '--preview', help='GUI to display the first second filtered with thresholds',
action='store_true')
parser.add_argument('-r', '--result', help='GUI to display the results on top of raw data',
action='store_true')
parser.add_argument('-s', '--second', type=int, default=0, help='If preview mode, begining of the preview [in s]')
parser.add_argument('-e', '--extension', help='extension to consider for merging, converting and deconverting',
default='None')
parser.add_argument('-o', '--output', help='output file [for generation of synthetic benchmarks]')
parser.add_argument('-t', '--type', help='benchmark type',
choices=['fitting', 'clustering', 'synchrony'])
if len(argv) == 0:
parser.print_help()
sys.exit(0)
args = parser.parse_args(argv)
steps = args.method.split(',')
for step in steps:
if step not in all_steps:
print_error(['The method "%s" is not recognized' % step])
sys.exit(0)
# To save some typing later
nb_gpu = 0
(nb_cpu, hostfile, batch, preview, result, extension, output, benchmark, info, second) = \
(args.cpu, args.hostfile, args.batch, args.preview, args.result, args.extension, args.output, args.type, args.info, args.second)
filename = os.path.abspath(args.datafile)
real_file = filename
f_next, extens = os.path.splitext(filename)
if info:
if args.datafile.lower() in __supported_data_files__:
filename = 'tmp'
if len(__supported_data_files__[args.datafile.lower()].extension) > 0:
filename += __supported_data_files__[args.datafile.lower()].extension[0]
__supported_data_files__[args.datafile.lower()](filename, {}, is_empty=True)._display_requirements_()
else:
print_and_log([
'',
'To get info on any particular file format, do:',
'>> spyking-circus file_format -i',
''
], 'default')
print_and_log(list_all_file_format())
sys.exit(0)
if extens == '.params':
print_error(['You should launch the code on the data file!'])
sys.exit(0)
file_params = f_next + '.params'
if not os.path.exists(file_params) and not batch:
print(Fore.RED + 'The parameter file %s is not present!' % file_params)
create_params = query_yes_no(Fore.WHITE + "Do you want SpyKING CIRCUS to create a parameter file?")
if create_params:
print(Fore.WHITE + "Creating %s" % file_params)
print(Fore.WHITE + "Fill it properly before launching the code! (see documentation)")
print_info(['Keep in mind that filtering is performed on site, so please',
'be sure to keep a copy of your data elsewhere'])
shutil.copyfile(config_file, file_params)
sys.exit(0)
elif batch:
tasks_list = filename
if not batch:
file_params = f_next + '.params'
if not os.path.exists(file_params):
print_and_log(["%s does not exist" % file_params], 'error')
sys.exit(0)
import ConfigParser as configparser
parser = configparser.ConfigParser()
myfile = open(file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
parser.read(file_params)
for section in CircusParser.__all_sections__:
if parser.has_section(section):
for (key, value) in parser.items(section):
parser.set(section, key, value.split('#')[0].rstrip())
else:
parser.add_section(section)
try:
use_output_dir = parser.get('data', 'output_dir') != ''
except Exception:
use_output_dir = False
if use_output_dir:
path = os.path.abspath(os.path.expanduser(parser.get('data', 'output_dir')))
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out):
os.makedirs(file_out)
else:
file_out = f_next
logfile = file_out + '.log'
if os.path.exists(logfile):
os.remove(logfile)
logger = init_logging(logfile)
params = CircusParser(filename)
data_file = params.get_data_file(source=True, has_been_created=False)
overwrite = params.getboolean('data', 'overwrite')
file_format = params.get('data', 'file_format')
if overwrite:
support_parallel_write = data_file.parallel_write
is_writable = data_file.is_writable
else:
support_parallel_write = __supported_data_files__['raw_binary'].parallel_write
is_writable = __supported_data_files__['raw_binary'].is_writable
if preview:
print_and_log(['Preview mode, showing only seconds [%d-%d] of the recording' % (second, second+1)], 'info', logger)
tmp_path_loc = os.path.join(os.path.abspath(params.get('data', 'file_out')), 'tmp')
if not os.path.exists(tmp_path_loc):
os.makedirs(tmp_path_loc)
filename = os.path.join(tmp_path_loc, 'preview.dat')
f_next, extens = os.path.splitext(filename)
preview_params = f_next + '.params'
shutil.copyfile(file_params, preview_params)
steps = ['filtering', 'whitening']
chunk_size = int(params.rate)
data_file.open()
nb_chunks, _ = data_file.analyze(chunk_size)
if nb_chunks <= (second + 1):
print_and_log(['Recording is too short to display seconds [%d-%d]' % (second, second+1)])
sys.exit(0)
local_chunk = data_file.get_snippet(int(second*params.rate), int(1.2*chunk_size))
description = data_file.get_description()
data_file.close()
new_params = CircusParser(filename, create_folders=False)
new_params.write('data', 'chunk_size', '1')
new_params.write('data', 'file_format', 'raw_binary')
new_params.write('data', 'data_dtype', 'float32')
new_params.write('data', 'data_offset', '0')
new_params.write('data', 'dtype_offset', '0')
new_params.write('data', 'stream_mode', 'None')
new_params.write('data', 'overwrite', 'True')
new_params.write('triggers', 'ignore_times', 'False')
new_params.write('data', 'sampling_rate', str(params.rate))
new_params.write('whitening', 'safety_time', '0')
new_params.write('clustering', 'safety_time', '0')
new_params.write('whitening', 'chunk_size', '1')
new_params.write('data', 'preview_path', params.file_params)
new_params.write('data', 'output_dir', '')
description['data_dtype'] = 'float32'
description['dtype_offset'] = 0
description['data_offset'] = 0
description['gain'] = 1.
new_params = CircusParser(filename)
data_file_out = new_params.get_data_file(is_empty=True, params=description)
support_parallel_write = data_file_out.parallel_write
is_writable = data_file_out.is_writable
data_file_out.allocate(shape=local_chunk.shape, data_dtype=numpy.float32)
data_file_out.open('r+')
data_file_out.set_data(0, local_chunk)
data_file_out.close()
if tasks_list is not None:
with open(tasks_list, 'r') as f:
for line in f:
if len(line) > 0:
subprocess.check_call(['spyking-circus'] + line.replace('\n', '').split(" "))
else:
print_and_log(['Config file: %s' % (f_next + '.params')], 'debug', logger)
print_and_log(['Data file : %s' % filename], 'debug', logger)
print(get_colored_header())
print(Fore.GREEN + "File : " + Fore.CYAN + real_file)
if preview:
print(Fore.GREEN + "Steps : " + Fore.CYAN + "preview mode")
elif result:
print(Fore.GREEN + "Steps : " + Fore.CYAN + "result mode")
else:
print(Fore.GREEN + "Steps : " + Fore.CYAN + ", ".join(steps))
# print Fore.GREEN + "GPU detected : ", Fore.CYAN + str(HAVE_CUDA)
print(Fore.GREEN + "Number of CPU : " + Fore.CYAN + str(nb_cpu) + "/" + str(psutil.cpu_count()))
# if HAVE_CUDA:
# print Fore.GREEN + "Number of GPU : ", Fore.CYAN + str(nb_gpu)
print(Fore.GREEN + "Parallel HDF5 : " + Fore.CYAN + str(parallel_hdf5))
do_upgrade = ''
use_shared_memory = get_shared_memory_flag(params)
if not SHARED_MEMORY:
do_upgrade = Fore.WHITE + ' [please consider upgrading MPI]'
print(Fore.GREEN + "Shared memory : " + Fore.CYAN + str(use_shared_memory) + do_upgrade)
print(Fore.GREEN + "Hostfile : " + Fore.CYAN + hostfile)
print("")
print(Fore.GREEN + "##################################################################")
print("")
print(Fore.RESET)
# Launch the subtasks
subtasks = [('filtering', 'mpirun'),
('whitening', 'mpirun'),
('clustering', 'mpirun'),
('fitting', 'mpirun'),
('extracting', 'mpirun'),
('gathering', 'python'),
('converting', 'mpirun'),
('deconverting', 'mpirun'),
('benchmarking', 'mpirun'),
('merging', 'mpirun'),
('validating', 'mpirun'),
('thresholding', 'mpirun')]
# if HAVE_CUDA and nb_gpu > 0:
# use_gpu = 'True'
# else:
use_gpu = 'False'
time = data_stats(params) / 60.0
if preview:
params = new_params
if nb_cpu < psutil.cpu_count():
if use_gpu != 'True' and not result:
print_and_log(['Using only %d out of %d local CPUs available (-c to change)' % (nb_cpu, psutil.cpu_count())], 'info', logger)
if params.getboolean('detection', 'matched-filter') and not params.getboolean('clustering', 'smart_search'):
print_and_log(['Smart Search should be activated for matched filtering'], 'info', logger)
if time > 30 and not params.getboolean('clustering', 'smart_search'):
print_and_log(['Smart Search should be activated for long recordings'], 'info', logger)
n_edges = get_averaged_n_edges(params)
if n_edges > 100 and not params.getboolean('clustering', 'compress'):
print_and_log(['Template compression is highly recommended based on parameters'], 'info', logger)
if not result:
for subtask, command in subtasks:
if subtask in steps:
if command == 'python':
# Directly call the launcher
try:
circus.launch(subtask, filename, nb_cpu, nb_gpu, use_gpu)
except:
print_and_log(['Step "%s" failed!' % subtask], 'error', logger)
sys.exit(0)
elif command == 'mpirun':
# Use mpirun to make the call
mpi_args = gather_mpi_arguments(hostfile, params)
one_cpu = False
if subtask in ['filtering', 'benchmarking'] and not is_writable:
if not preview and overwrite:
print_and_log(['The file format %s is read only!' % file_format,
'You should set overwite to False, to create a copy of the data.',
'However, note that if you have streams, informations on times',
'will be discarded'], 'info', logger)
sys.exit(0)
if subtask in ['filtering'] and not support_parallel_write and (args.cpu > 1):
print_and_log(['No parallel writes for %s: only 1 node used for %s' %(file_format, subtask)], 'info', logger)
nb_tasks = str(1)
one_cpu = True
else:
if subtask != 'fitting':
nb_tasks = str(args.cpu)
else:
# if use_gpu == 'True':
# nb_tasks = str(args.gpu)
# else:
nb_tasks = str(args.cpu)
if subtask == 'benchmarking':
if (output is None) or (benchmark is None):
print_and_log(["To generate synthetic datasets, you must provide output and type"], 'error', logger)
sys.exit(0)
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, output, benchmark
]
elif subtask in ['merging', 'converting']:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, extension
]
elif subtask in ['deconverting']:
nb_tasks = str(1)
nb_cpu = 1
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask', subtask,
filename, str(nb_cpu), str(nb_gpu), use_gpu,
extension
]
else:
mpi_args += [
'-np', nb_tasks, 'spyking-circus-subtask',
subtask, filename, str(nb_cpu), str(nb_gpu),
use_gpu, str(one_cpu)
]
print_and_log(['Launching task %s' % subtask], 'debug', logger)
print_and_log(['Command: %s' % str(mpi_args)], 'debug', logger)
try:
subprocess.check_call(mpi_args)
except subprocess.CalledProcessError as e:
print_and_log(['Step "%s" failed for reason %s!' % (subtask, e)], 'error', logger)
sys.exit(0)
if preview or result:
from circus.shared import gui
import pylab
try:
from PyQt5.QtWidgets import QApplication
except ImportError:
from matplotlib.backends import qt_compat
use_pyside = qt_compat.QT_API == qt_compat.QT_API_PYSIDE
if use_pyside:
from PySide.QtGui import QApplication
else:
from PyQt4.QtGui import QApplication
app = QApplication([])
try:
pylab.style.use('ggplot')
except Exception:
pass
if preview:
print_and_log(['Launching the preview GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(new_params)
shutil.rmtree(tmp_path_loc)
elif result:
data_file = params.get_data_file()
print_and_log(['Launching the result GUI...'], 'debug', logger)
mygui = gui.PreviewGUI(params, show_fit=True)
sys.exit(app.exec_())
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to concatenate artefacts/dead times before using
stream mode. Code will look for .dead and .trig files, and
concatenate them automatically taking care of file offsets
'''
parser = argparse.ArgumentParser(
description=header, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('datafile', help='data file')
# parser.add_argument('-w', '--window', help='text file with artefact window files',
# default=None)
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
# if args.window is None:
# window_file = None
# else:
# window_file = os.path.abspath(args.window)
filename = os.path.abspath(args.datafile)
params = CircusParser(filename)
dead_in_ms = params.getboolean('triggers', 'dead_in_ms')
trig_in_ms = params.getboolean('triggers', 'trig_in_ms')
if os.path.exists(params.logfile):
os.remove(params.logfile)
_ = init_logging(params.logfile)
logger = logging.getLogger(__name__)
if params.get('data', 'stream_mode') == 'multi-files':
data_file = params.get_data_file(source=True, has_been_created=False)
all_times_dead = numpy.zeros((0, 2), dtype=numpy.int64)
all_times_trig = numpy.zeros((0, 2), dtype=numpy.int64)
for f in data_file._sources:
name, ext = os.path.splitext(f.file_name)
dead_file = f.file_name.replace(ext, '.dead')
trig_file = f.file_name.replace(ext, '.trig')
if os.path.exists(dead_file):
print_and_log(['Found file %s' % dead_file], 'default', logger)
times = get_dead_times(dead_file, data_file.sampling_rate,
dead_in_ms)
if times.max() > f.duration or times.min() < 0:
print_and_log([
'Dead zones larger than duration for file %s' %
f.file_name, '-> Clipping automatically'
], 'error', logger)
times = numpy.minimum(times, f.duration)
times = numpy.maximum(times, 0)
times += f.t_start
all_times_dead = numpy.vstack((all_times_dead, times))
if os.path.exists(trig_file):
print_and_log(['Found file %s' % trig_file], 'default', logger)
times = get_trig_times(trig_file, data_file.sampling_rate,
trig_in_ms)
if times[:, 1].max() > f.duration or times[:, 1].min() < 0:
print_and_log([
'Triggers larger than duration for file %s' %
f.file_name
], 'error', logger)
sys.exit(0)
times[:, 1] += f.t_start
all_times_trig = numpy.vstack((all_times_trig, times))
if len(all_times_dead) > 0:
output_file = os.path.join(os.path.dirname(filename),
'dead_zones.txt')
print_and_log(['Saving global artefact file in %s' % output_file],
'default', logger)
if dead_in_ms:
all_times_dead = all_times_dead.astype(
numpy.float32) / data_file.sampling_rate
numpy.savetxt(output_file, all_times_dead)
if len(all_times_trig) > 0:
output_file = os.path.join(os.path.dirname(filename),
'triggers.txt')
print_and_log(['Saving global artefact file in %s' % output_file],
'default', logger)
if trig_in_ms:
all_times_trig = all_times_trig.astype(
numpy.float32) / data_file.sampling_rate
numpy.savetxt(output_file, all_times_trig)
elif params.get('data', 'stream_mode') == 'single-file':
print_and_log(['Not implemented'], 'error', logger)
sys.exit(0)
else:
print_and_log(
['You should select a valid stream_mode such as multi-files'],
'error', logger)
sys.exit(0)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
header = get_colored_header()
header += '''Utility to group files within several folders into a single
virtual folder, such that they can be processed together with the
multi-files mode.
If you want to also process .dead or .trig files in order to later
on concatenate artefacts, please use the -d or -t options
'''
parser = argparse.ArgumentParser(description=header,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('folders', help='text file with the list of folders to consider')
parser.add_argument('extension', help='file extension to consider within folders')
parser.add_argument('-o', '--output', help='name of the output folder [default is output]', default='output')
parser.add_argument('-d', '--dead', help='Search for all .dead files', action='store_true')
parser.add_argument('-t', '--trig', help='Search for all .trig files', action='store_true')
if len(argv) == 0:
parser.print_help()
sys.exit()
args = parser.parse_args(argv)
folders_file = os.path.abspath(args.folders)
output = os.path.abspath(args.output)
extension = args.extension
filename, ext = os.path.splitext(os.path.basename(folders_file))
logger = init_logging(filename + '.log')
logger = logging.getLogger(__name__)
if not os.path.exists(folders_file):
print_and_log(['The folder file %s does not exists!' %folders_file], 'error', logger)
sys.exit(0)
try:
folders = []
myfile = open(folders_file, 'r')
lines = myfile.readlines()
myfile.close()
for l in lines:
folders += [os.path.abspath(l.strip())]
except Exception:
print_and_log(['Check the syntax of the folder file'], 'error', logger)
sys.exit(0)
do_folders = True
if os.path.exists(output):
do_folders = query_yes_no(Fore.WHITE + "Folder %s already exists! Do you want to erase everything?" %output, default=None)
if not do_folders:
sys.exit(0)
else:
shutil.rmtree(output)
os.makedirs(output)
for count, folder in enumerate(folders):
files = os.listdir(folder)
for file in files:
_, ext = os.path.splitext(file)
ext = ext.strip('.')
if (ext.lower() == extension.lower()) or (args.dead and ext.lower() == 'dead') or (args.trig and ext.lower()== 'trig'):
original_file = os.path.join(folder, file)
linked_file = os.path.join(output, 'sc_{c}_{f}'.format(c=count, f=os.path.basename(original_file)))
if not os.path.exists(linked_file):
os.symlink(original_file, linked_file)
else:
os.symlink(original_file, linked_file)
