def gather_mpi_arguments(hostfile, params):
print_and_log(['MPI detected: %s' % str(MPI_VENDOR)], 'debug', logger)
if MPI_VENDOR[0] == 'Open MPI':
mpi_args = ['mpirun']
if os.getenv('LD_LIBRARY_PATH'):
mpi_args += ['-x', 'LD_LIBRARY_PATH']
if os.getenv('PATH'):
mpi_args += ['-x', 'PATH']
if os.getenv('PYTHONPATH'):
mpi_args += ['-x', 'PYTHONPATH']
if os.path.exists(hostfile):
mpi_args += ['-hostfile', hostfile]
elif MPI_VENDOR[0] == 'Microsoft MPI':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-machinefile', hostfile]
elif MPI_VENDOR[0] == 'MPICH2':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-f', hostfile]
elif MPI_VENDOR[0] == 'MPICH':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-f', hostfile]
else:
print_and_log([
'%s may not be yet properly implemented: contact developpers' %
MPI_VENDOR[0]
], 'error', logger)
mpi_args = ['mpirun']
if os.path.exists(hostfile):
mpi_args += ['-hostfile', hostfile]
return mpi_args
def getboolean(self, section, data, check=True):
"""
Gets the boolean variable from the variable data in a section
Parameters
----------
section : str
the section in *params to be read (e.g., 'filtering')
data : str
the variable data to be read (e.g., 'remove_median')
Returns
-------
bool
if the variable data is applied or not.
"""
if check is False:
return self.parser.getboolean(section, data)
try:
return self.parser.getboolean(section, data)
except Exception:
if comm.rank == 0:
print_and_log([
"Parameter %s is missing in section [%s]" % (data, section)
], 'error', logger)
sys.exit(0)
def getint(self, section, data, check=True):
"""
Gets the value of the variable data in a section
Parameters
----------
section : str
the section in *params to be read (e.g., 'detection')
data : str
the variable data to be read (e.g., 'oversampling_factor')
Returns
-------
int
The value of the variable data.
"""
if check is False:
return self.parser.getint(section, data)
try:
return self.parser.getint(section, data)
except Exception:
if comm.rank == 0:
print_and_log([
"Parameter %s is missing in section [%s]" % (data, section)
], 'error', logger)
sys.exit(0)
def _create_data_file(self, data_file, is_empty, params, stream_mode):
file_format = params.pop('file_format').lower()
if comm.rank == 0:
print_and_log(
['Trying to read file %s as %s' % (data_file, file_format)],
'debug', logger)
data = __supported_data_files__[file_format](data_file, params,
is_empty, stream_mode)
self.rate = data.sampling_rate
self.nb_channels = data.nb_channels
self.gain = data.gain
self.data_file = data
self._update_rate_values()
N_e = self.getint('data', 'N_e')
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'Analyzed %d channels but only %d are recorded' %
(N_e, self.nb_channels)
], 'error', logger)
sys.exit(0)
return data
def purge(file, pattern):
dir = os.path.dirname(os.path.abspath(file))
for f in os.listdir(dir):
if f.find(pattern) > -1:
os.remove(os.path.join(dir, f))
if comm.rank == 0:
print_and_log(['Removing %s for directory %s' %(pattern, dir)], 'debug', logger)
def test_mpi_ring(nb_nodes):
if comm.size != nb_nodes:
if comm.rank == 0:
print_and_log([
'The MPI install does not seems to be correct!',
'Be sure mpi4py has been compiled for your default version!'
], 'warning', logger)
sys.exit()
def _update_rate_values(self):
if self._N_t is None:
if comm.rank == 0:
print_and_log(
['Changing all values in the param depending on the rate'],
'debug', logger)
try:
self._N_t = self.getfloat('detection', 'N_t')
except Exception:
if comm.rank == 0:
print_and_log(
['N_t must now be defined in the [detection] section'],
'error', logger)
sys.exit(0)
self._N_t = int(self.rate * self._N_t * 1e-3)
jitter_range = self.getfloat('detection', 'jitter_range')
self.set('detection', 'jitter_range',
str(int(self.rate * jitter_range * 1e-3)))
if numpy.mod(self._N_t, 2) == 0:
self._N_t += 1
self.set('detection', 'N_t', str(self._N_t))
self.set('detection', 'dist_peaks', str(self._N_t))
self.set('detection', 'template_shift', str((self._N_t - 1) // 2))
self._savgol = int(self.rate * 0.5 * 1e-3)
if numpy.mod(self._savgol, 2) == 0:
self._savgol += 1
self.set('clustering', 'savgol_window', str(self._savgol))
if self.parser._sections['fitting'].has_key('chunk'):
self.parser.set('fitting', 'chunk_size',
self.parser._sections['fitting']['chunk'])
for section in ['data', 'whitening', 'fitting']:
chunk_size = int(
self.parser.getfloat(section, 'chunk_size') * self.rate)
self.set(section, 'chunk_size', str(chunk_size))
for section in ['clustering', 'whitening', 'extracting']:
safety_time = self.get(section, 'safety_time')
if safety_time == 'auto':
self.set(section, 'safety_time', str(self._N_t // 3))
else:
safety_time = float(safety_time)
self.set(section, 'safety_time',
str(int(safety_time * self.rate * 1e-3)))
refractory = self.getfloat('fitting', 'refractory')
self.set('fitting', 'refractory',
str(int(refractory * self.rate * 1e-3)))
def write(self, section, flag, value, preview_path=False):
if comm.rank == 0:
print_and_log(
['Writing value %s for %s:%s' % (value, section, flag)],
'debug', logger)
self.parser.set(section, flag, value)
if preview_path:
f = open(self.get('data', 'preview_path'), 'r')
else:
f = open(self.file_params, 'r')
lines = f.readlines()
f.close()
spaces = ''.join([' '] * (max(0, 15 - len(flag))))
to_write = '%s%s= %s #!! AUTOMATICALLY EDITED: DO NOT MODIFY !!\n' % (
flag, spaces, value)
section_area = [0, len(lines)]
idx = 0
for count, line in enumerate(lines):
if (idx == 1) and line.strip().replace('[', '').replace(
']', '') in self.__all_sections__:
section_area[idx] = count
idx += 1
if (line.find('[%s]' % section) > -1):
section_area[idx] = count
idx += 1
has_been_changed = False
for count in xrange(section_area[0] + 1, section_area[1]):
if '=' in lines[count]:
key = lines[count].split('=')[0].replace(' ', '')
if key == flag:
lines[count] = to_write
has_been_changed = True
if not has_been_changed:
lines.insert(section_area[1] - 1, to_write)
if preview_path:
f = open(self.get('data', 'preview_path'), 'w')
else:
f = open(self.file_params, 'w')
for line in lines:
f.write(line)
f.close()
def read_probe(parser):
probe = {}
filename = os.path.abspath(
os.path.expanduser(parser.get('data', 'mapping')))
if comm.rank == 0:
print_and_log(["Reading the probe file %s" % filename], 'debug',
logger)
if not os.path.exists(filename):
if comm.rank == 0:
print_and_log(["The probe file %s can not be found" % filename],
'error', logger)
sys.exit(1)
try:
with open(filename, 'r') as f:
probetext = f.read()
exec(probetext, probe)
except Exception as ex:
if comm.rank == 0:
print_and_log([
"Something wrong with the syntax of the probe file:\n" +
str(ex)
], 'error', logger)
sys.exit(1)
key_flags = ['total_nb_channels', 'radius', 'channel_groups']
for key in key_flags:
if not probe.has_key(key):
if comm.rank == 0:
print_and_log(["%s is missing in the probe file" % key],
'error', logger)
sys.exit(1)
return probe
def parse_dead_channels(channels):
is_correct = False
try:
dead_channels = ast.literal_eval(channels)
is_correct = True
except Exception:
pass
if not is_correct:
if comm.rank == 0:
print_and_log(["The syntax for dead channels is not correct!"],
'error', logger)
sys.exit(0)
else:
return dead_channels
def test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
template_file = file_out_suff + '.templates%s.hdf5' % extension
if os.path.exists(template_file):
version = io.load_data(params, 'version', extension)
else:
print_and_log(['No templates found! Check suffix?'], 'error', logger)
sys.exit(0)
if version is not None:
if (StrictVersion(version) >= StrictVersion('0.6.0')):
return True
else:
print_and_log(["Version is below 0.6.0"], 'debug', logger)
return False
def get_data_file(self, is_empty=False, params=None, source=False, has_been_created=True):
if params is None:
params = {}
for key, value in self.parser._sections['data'].items():
if key not in params:
params[key] = value
data_file = params.pop('data_file')
stream_mode = self.get('data', 'stream_mode').lower()
if stream_mode in ['none']:
stream_mode = None
if not self.getboolean('data', 'overwrite'):
# If we do not want to overwrite, we first read the original data file
# Then, if we do not want to obtain it as a source file, we switch the
# format to raw_binary and the output file name
if not source:
# First we read the original data file, that should not be empty
print_and_log(['Reading first the real data file to get the parameters'], 'debug', logger)
tmp = self._create_data_file(data_file, False, params, stream_mode)
# Then we change the dataa_file name
data_file = self.get('data', 'data_file_no_overwrite')
if comm.rank == 0:
print_and_log(['Forcing the exported data file to be of type raw_binary'], 'debug', logger)
# And we force the results to be of type float32, without streams
params['file_format'] = 'raw_binary'
params['data_dtype'] = 'float32'
params['dtype_offset'] = 0
params['data_offset'] = 0
params['sampling_rate'] = self.rate
params['nb_channels'] = self.nb_channels
params['gain'] = self.gain
stream_mode = None
data_file, extension = os.path.splitext(data_file)
data_file += ".dat"
else:
if has_been_created:
data_file = self.get('data', 'data_file_no_overwrite')
if not os.path.exists(data_file):
if comm.rank== 0:
print_and_log(['The overwrite option is only valid if the filtering step is launched before!'], 'error', logger)
sys.exit(1)
else:
if comm.rank== 0:
print_and_log(['The copy file has not yet been created! Returns normal file'], 'debug', logger)
return self._create_data_file(data_file, is_empty, params, stream_mode)
def _create_data_file(self, data_file, is_empty, params, stream_mode):
"""
Creates a data file with parameters from the param file.
Parameters
----------
data_file : str
the path for the datafile
is_empty : bool
params : dict
stream_mode : str
multi-files, multi-folders or single-file.
Returns
-------
dict
a supported data file with the parameters from the param file.
"""
file_format = params.pop('file_format').lower()
if comm.rank == 0:
print_and_log(
['Trying to read file %s as %s' % (data_file, file_format)],
'debug', logger)
data = __supported_data_files__[file_format](data_file, params,
is_empty, stream_mode)
self.rate = data.sampling_rate
self.nb_channels = data.nb_channels
self.gain = data.gain
self.data_file = data
self._update_rate_values()
N_e = self.getint('data', 'N_e')
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'Analyzed %d channels but only %d are recorded' %
(N_e, self.nb_channels)
], 'error', logger)
sys.exit(0)
return data
def detect_memory(params, whitening=False, filtering=False, fitting=False):
from psutil import virtual_memory
N_e = params.getint('data', 'N_e')
safety_threshold = params.getfloat('data', 'memory_usage')
data_file = params.data_file
data_file.open()
sampling_rate = data_file.sampling_rate
duation = data_file.duration
data_file.close()
from uuid import getnode as get_mac
myip = numpy.int64(get_mac()) % 100000
sub_comm = comm.Split_type(MPI.COMM_TYPE_SHARED, myip)
res = numpy.zeros(1, dtype=numpy.int64)
mem = virtual_memory()
if sub_comm.rank == 0:
res[0] = safety_threshold * numpy.int64(mem.available // sub_comm.size)
sub_comm.Barrier()
sub_comm.Free()
memory = all_gather_array(res, comm, 1, 'int64')
idx = numpy.where(memory > 0)
max_memory = numpy.min(memory[idx]) // (4 * N_e)
if whitening or filtering:
max_size = int(30 * data_file.sampling_rate)
elif fitting:
max_size = int(0.1 * data_file.sampling_rate)
else:
max_size = (data_file.duration // comm.size)
chunk_size = min(max_memory, max_size)
if comm.rank == 0:
print_and_log([
'Setting data chunk size to %g second' %
(chunk_size / float(sampling_rate))
], 'debug', logger)
return chunk_size
def read_probe(parser, radius_in_probe=True):
"""
Read the probe file
Arguments
---------
parser :
The circur parser object.
radius_in_probe: bool
True (default) if radius is read from the probe.
False if is read from params file
Returns
-------
The probe
"""
probe = {}
filename = os.path.abspath(
os.path.expanduser(parser.get('data', 'mapping')))
if comm.rank == 0:
print_and_log(["Reading the probe file %s" % filename], 'debug',
logger)
if not os.path.exists(filename):
if comm.rank == 0:
print_and_log(["The probe file %s can not be found" % filename],
'error', logger)
sys.exit(0)
try:
with open(filename, 'r') as f:
probetext = f.read()
exec(probetext, probe)
del probe['__builtins__']
except Exception as ex:
if comm.rank == 0:
print_and_log([
"Something wrong with the syntax of the probe file:\n" +
str(ex)
], 'error', logger)
sys.exit(0)
key_flags = ['total_nb_channels', 'channel_groups']
if radius_in_probe:
key_flags += ['radius']
for key in key_flags:
if not probe.has_key(key):
if comm.rank == 0:
print_and_log(["%s is missing in the probe file" % key],
'error', logger)
sys.exit(0)
return probe
def test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
template_file = file_out_suff + '.templates%s.hdf5' % extension
if os.path.exists(template_file):
try:
version = h5py.File(
template_file, 'r',
libver='latest').get('version')[0].decode('ascii')
except Exception:
version = None
else:
raise Exception('No templates found! Check suffix?')
if version is not None:
if (StrictVersion(version) >= StrictVersion('0.6.0')):
return True
else:
print_and_log(["Version is below 0.6.0"], 'debug', logger)
return False
def detect_memory(params, safety_threshold=0.1):
from psutil import virtual_memory
N_e = params.getint('data', 'N_e')
data_file = params.data_file
data_file.open()
sampling_rate = data_file.sampling_rate
duation = data_file.duration
data_file.close()
from uuid import getnode as get_mac
myip = numpy.int64(get_mac()) % 100000
sub_comm = comm.Split_type(MPI.COMM_TYPE_SHARED, myip)
res = numpy.zeros(1, dtype=numpy.int64)
mem = virtual_memory()
if sub_comm.rank == 0:
res[0] = safety_threshold * numpy.int64(mem.available // sub_comm.size)
sub_comm.Barrier()
sub_comm.Free()
memory = all_gather_array(res, comm, 1, 'int64')
idx = numpy.where(memory > 0)
max_memory = numpy.min(memory[idx]) // (4 * N_e)
max_size = (data_file.duration // comm.size)
for section in ['data', 'whitening']:
chunk_size = min(max_memory, max_size)
params.set(section, 'chunk_size', str(chunk_size))
if comm.rank == 0:
print_and_log([
'Setting data chunk size to %d second' %
(max_size / float(sampling_rate))
], 'info', logger)
return params
def gather_mpi_arguments(hostfile, params):
print_and_log(['MPI detected: %s' % str(MPI_VENDOR)], 'debug', logger)
if MPI_VENDOR[0] == 'Open MPI':
if MPI_VENDOR[1][0] >= 3:
print_and_log([
'SpyKING CIRCUS does not work with OPENMPI >= 3.0',
'Consider downgrading or switching to MPICH'
], 'error', logger)
sys.exit(0)
mpi_args = ['mpirun']
if os.getenv('LD_LIBRARY_PATH'):
mpi_args += ['-x', 'LD_LIBRARY_PATH']
if os.getenv('PATH'):
mpi_args += ['-x', 'PATH']
if os.getenv('PYTHONPATH'):
mpi_args += ['-x', 'PYTHONPATH']
if os.path.exists(hostfile):
mpi_args += ['-hostfile', hostfile]
elif MPI_VENDOR[0] == 'Microsoft MPI':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-machinefile', hostfile]
elif MPI_VENDOR[0] == 'MPICH2':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-f', hostfile]
elif MPI_VENDOR[0] == 'MPICH':
mpi_args = ['mpiexec']
if os.path.exists(hostfile):
mpi_args += ['-f', hostfile]
else:
print_and_log([
'%s may not be yet properly implemented: contact developpers' %
MPI_VENDOR[0]
], 'error', logger)
mpi_args = ['mpirun']
if os.path.exists(hostfile):
mpi_args += ['-hostfile', hostfile]
return mpi_args
def apply_patch_for_similarities(params, extension):
if not test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
hdf5_compress = params.getboolean('data', 'hdf5_compress')
blosc_compress = params.getboolean('data', 'blosc_compress')
N_tm = io.load_data(params, 'nb_templates', extension)
N_half = int(N_tm // 2)
N_t = params.getint('detection', 'N_t')
duration = 2 * N_t - 1
if comm.rank == 0:
print_and_log(["Fixing overlaps from 0.5.XX..."], 'default',
logger)
maxlag = numpy.zeros((N_half, N_half), dtype=numpy.int32)
maxoverlap = numpy.zeros((N_half, N_half), dtype=numpy.float32)
to_explore = numpy.arange(N_half - 1)[comm.rank::comm.size]
if comm.rank == 0:
to_explore = get_tqdm_progressbar(to_explore)
if not SHARED_MEMORY:
over_x, over_y, over_data, over_shape = io.load_data(
params, 'overlaps-raw', extension=extension)
else:
over_x, over_y, over_data, over_shape = io.load_data_memshared(
params, 'overlaps-raw', extension=extension)
for i in to_explore:
idx = numpy.where((over_x >= i * N_tm + i + 1)
& (over_x < (i * N_tm + N_half)))[0]
local_x = over_x[idx] - (i * N_tm + i + 1)
data = numpy.zeros((N_half - (i + 1), duration),
dtype=numpy.float32)
data[local_x, over_y[idx]] = over_data[idx]
maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1)
maxlag[i + 1:, i] = -maxlag[i, i + 1:]
maxoverlap[i, i + 1:] = numpy.max(data, 1)
maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:]
#Now we need to sync everything across nodes
maxlag = gather_array(maxlag,
comm,
0,
1,
'int32',
compress=blosc_compress)
if comm.rank == 0:
maxlag = maxlag.reshape(comm.size, N_half, N_half)
maxlag = numpy.sum(maxlag, 0)
maxoverlap = gather_array(maxoverlap,
comm,
0,
1,
'float32',
compress=blosc_compress)
if comm.rank == 0:
maxoverlap = maxoverlap.reshape(comm.size, N_half, N_half)
maxoverlap = numpy.sum(maxoverlap, 0)
if comm.rank == 0:
myfile2 = h5py.File(file_out_suff +
'.templates%s.hdf5' % extension,
'r+',
libver='earliest')
for key in ['maxoverlap', 'maxlag', 'version']:
if key in myfile2.keys():
myfile2.pop(key)
myfile2.create_dataset('version',
data=numpy.array(
circus.__version__.split('.'),
dtype=numpy.int32))
if hdf5_compress:
myfile2.create_dataset('maxlag',
data=maxlag,
compression='gzip')
myfile2.create_dataset('maxoverlap',
data=maxoverlap,
compression='gzip')
else:
myfile2.create_dataset('maxlag', data=maxlag)
myfile2.create_dataset('maxoverlap', data=maxoverlap)
myfile2.close()
def __init__(self, file_name, **kwargs):
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.logfile = f_next + '.log'
self.parser = configparser.ConfigParser()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" %self.file_params], 'error', logger)
sys.exit(1)
if comm.rank == 0:
print_and_log(['Creating a Circus Parser for datafile %s' %self.file_name], 'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in kwargs.items():
for section in self.__all_sections__:
if self.parser._sections[section].has_key(key):
self.parser._sections[section][key] = value
self.probe = read_probe(self.parser)
N_e = 0
for key in self.probe['channel_groups'].keys():
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log(['The number of analyzed channels is higher than the number of recorded channels'], 'error', logger)
sys.exit(1)
to_write = ["You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", '']
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]
to_write += ['', "To get more info on a given file format, see",
">> spyking-circus file_format -i"]
print_and_log(to_write, 'error', logger)
sys.exit(1)
test = self.file_format.lower() in __supported_data_files__.keys()
if not test:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]
to_write += ['', "To get more info on a given file format, see",
">> spyking-circus file_format -i"]
print_and_log(to_write, 'error', logger)
sys.exit(1)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers', 'trig_file') == '') or (self.parser.get('triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log(["trig_file and trig_windows must be specified in [triggers]"], 'error', logger)
sys.exit(1)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(["trig_unit in [triggers] should be in %s" %str(units)], 'error', logger)
sys.exit(1)
else:
self.parser.set('triggers', 'trig_in_ms', str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
self.parser.set('triggers', 'trig_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
self.parser.set('triggers', 'trig_windows', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_windows'))))
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(["dead_unit in [triggers] should be in %s" %str(units)], 'error', logger)
sys.exit(1)
else:
self.parser.set('triggers', 'dead_in_ms', str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
self.parser.set('triggers', 'dead_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'dead_file'))))
test = (self.parser.get('clustering', 'extraction').lower() in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log(["Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"], 'error', logger)
sys.exit(1)
test = (self.parser.get('detection', 'peaks').lower() in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log(["Only 3 detection modes for peaks in [detection]: negative, positive, both"], 'error', logger)
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
self.parser.set('data', 'data_file_no_overwrite', os.path.join(file_path, os.path.basename(f_next) + '_all_sc.dat'))
file_out = os.path.join(f_next, os.path.basename(f_next))
self.parser.set('data', 'file_out', file_out) # Output file without suffix
self.parser.set('data', 'file_out_suff', file_out + self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext', f_next) # Data file (assuming .filtered at the end)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') > 0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["nb_elts in [%s] should be in [0,1]" %section], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'noise_thr') >= 0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('validating', 'test_size') > 0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'cc_merge') >= 0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('merging', 'auto_mode') >= 0) and (self.parser.getfloat('merging', 'auto_mode') < 1)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be in [0,1["], 'error', logger)
sys.exit(1)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering', 'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log(["make_plots in [%s] should be in %s" %(section, str(fileformats))], 'error', logger)
sys.exit(1)
dispersion = self.parser.get('clustering', 'dispersion').replace('(', '').replace(')', '').split(',')
dispersion = map(float, dispersion)
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log(["min and max dispersions in [clustering] should be positive"], 'error', logger)
sys.exit(1)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting', 'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log(["export_pcs in [converting] should be in %s" %str(pcs_export)], 'error', logger)
sys.exit(1)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
def __init__(self, file_name, **kwargs):
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.logfile = f_next + '.log'
self.parser = configparser.ConfigParser()
## First, we remove all tabulations from the parameter file, in order
## to secure the parser
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in list(kwargs.items()):
for section in self.__all_sections__:
if key in self.parser._sections[section]:
self.parser._sections[section][key] = value
self.probe = read_probe(self.parser)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in list(self.probe['channel_groups'].keys()):
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in list(
__supported_data_files__.keys())
if not test:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'trig_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
self.parser.set(
'triggers', 'trig_windows',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers',
'trig_windows'))))
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'dead_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file'))))
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out):
os.makedirs(file_out)
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'noise_thr') >=
0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >= 0)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be positive"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be postiive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = list(map(float, dispersion))
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
def __init__(self, file_name, create_folders=True, **kwargs):
"""
Parameters:
----------
file_name : string
a path containing the *params file.
create_folders : bool
if a folder will be created. If true, output_dir in the data section
of the param file will be created.
Returns:
--------
a CircusParser object.
"""
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.do_folders = create_folders
self.parser = configparser.ConfigParser()
valid_path = check_valid_path(self.file_params)
if not valid_path:
print_and_log(
["Not all nodes can read/write the data file. Check path?"],
'error', logger)
sys.exit(0)
## First, we remove all tabulations from the parameter file, in order
## to secure the parser
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in kwargs.items():
for section in self.__all_sections__:
if self.parser._sections[section].has_key(key):
self.parser._sections[section][key] = value
if self.do_folders and self.parser.get('data', 'output_dir') == '':
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out) and self.do_folders:
os.makedirs(file_out)
self.logfile = file_out + '.log'
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.logfile = f_next + '.log'
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
# read .prb file
try:
self.parser.get('detection', 'radius')
except Exception: # radius == auto by default
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception: # when radius = auto in params file
self.probe = read_probe(self.parser, radius_in_probe=True)
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
else:
self.probe = read_probe(self.parser, radius_in_probe=False)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in self.probe['channel_groups'].keys():
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in __supported_data_files__.keys()
if not test:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'clean_artefact'):
for key in ['trig_file', 'trig_windows']:
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', key)))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(
["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', key, myfile)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'ignore_times'):
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file')))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', 'dead_file', myfile)
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
sys.exit(0)
common_ground = self.parser.get('filtering', 'common_ground')
if common_ground != '':
try:
self.parser.set('filtering', 'common_ground',
str(int(common_ground)))
except Exception:
self.parser.set('filtering', 'common_ground', '-1')
else:
self.parser.set('filtering', 'common_ground', '-1')
common_ground = self.parser.getint('filtering', 'common_ground')
all_electrodes = []
for key in self.probe['channel_groups'].keys():
all_electrodes += self.probe['channel_groups'][key]['channels']
test = (common_ground == -1) or common_ground in all_electrodes
if not test:
if comm.rank == 0:
print_and_log([
"Common ground in filtering section should be a valid electrode"
], 'error', logger)
sys.exit(0)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'noise_thr') >=
0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('data', 'memory_usage') >
0) and (self.parser.getfloat('data', 'memory_usage') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["memory_usage in [data] should be in ]0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >=
0) and (self.parser.getfloat('merging', 'auto_mode') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be in [0, 1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'noise_limit') >= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_limit in [merging] should be > 1"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'sparsity_limit') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["sparsity_limit in [merging] should be < 1"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be positive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering']:
test = self.parser.get('clustering',
'debug_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"debug_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
methods = [
'distance', 'dip', 'folding', 'nd-folding', 'bhatta', 'nd-bhatta'
]
test = self.parser.get('clustering',
'merging_method').lower() in methods
if not test:
if comm.rank == 0:
print_and_log([
"merging_method in [%s] should be in %s" %
(section, str(methods))
], 'error', logger)
sys.exit(0)
if self.parser.get('clustering', 'merging_param').lower() == 'default':
method = self.parser.get('clustering', 'merging_method').lower()
if method == 'dip':
self.parser.set('clustering', 'merging_param', '0.5')
elif method == 'distance':
self.parser.set('clustering', 'merging_param', '3')
elif method in ['folding', 'nd-folding']:
self.parser.set('clustering', 'merging_param', '1e-9')
elif method in ['bhatta', 'nd-bhatta']:
self.parser.set('clustering', 'merging_param', '2')
has_same_elec = self.parser.has_option('clustering', 'sim_same_elec')
has_dip_thresh = self.parser.has_option('clustering', 'dip_threshold')
if has_dip_thresh:
dip_threshold = self.parser.getfloat('clustering', 'dip_threshold')
if has_dip_thresh and (dip_threshold > 0):
if comm.rank == 0:
print_and_log([
"dip_threshold in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"By default a distance merging method with param 3 is assumed"
], 'info', logger)
self.parser.set('clustering', 'merging_param', str(3))
self.parser.set('clustering', 'merging_method', 'distance')
elif has_same_elec:
sim_same_elec = self.parser.get('clustering', 'sim_same_elec')
if comm.rank == 0:
print_and_log([
"sim_same_elec in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"Meanwhile a distance merging method with param %s is assumed"
% sim_same_elec
], 'info', logger)
self.parser.set('clustering', 'merging_param', sim_same_elec)
self.parser.set('clustering', 'merging_method', 'distance')
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = map(float, dispersion)
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
if self.parser.getboolean('detection', 'hanning'):
if comm.rank == 0:
print_and_log(["Hanning filtering is activated"], 'debug',
logger)
def apply_patch_for_similarities(params, extension):
if not test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
if comm.rank == 0:
print_and_log(["Fixing overlaps from 0.5.XX..."], 'default',
logger)
over_file = file_out_suff + '.overlap.hdf5'
if os.path.exists(over_file):
over_x = h5py.File(over_file, 'r',
libver='latest').get('over_x')[:].ravel()
over_y = h5py.File(over_file, 'r',
libver='latest').get('over_y')[:].ravel()
over_data = h5py.File(
over_file, 'r',
libver='latest').get('over_data')[:].ravel()
over_shape = h5py.File(
over_file, 'r',
libver='latest').get('over_shape')[:].ravel()
overlap = scipy.sparse.csc_matrix(
(over_data, (over_x, over_y)), shape=over_shape)
else:
raise Exception('No overlaps found! Check suffix?')
myfile2 = h5py.File(file_out_suff +
'.templates%s.hdf5' % extension,
'r+',
libver='latest')
N_tm = int(numpy.sqrt(overlap.shape[0]))
N_t = params.getint('detection', 'N_t')
if 'maxoverlap' in myfile2.keys():
maxoverlap = myfile2['maxoverlap']
else:
maxoverlap = myfile2.create_dataset('maxoverlap',
shape=(N_tm // 2,
N_tm // 2),
dtype=numpy.float32)
if 'maxlag' in myfile2.keys():
maxlag = myfile2['maxlag']
else:
maxlag = myfile2.create_dataset('maxlag',
shape=(N_tm // 2, N_tm // 2),
dtype=numpy.int32)
if 'version' in myfile2.keys():
version = myfile2['version']
else:
version = myfile2.create_dataset(
'version',
data=numpy.array(
[circus.__version__.encode('ascii', 'ignore')]))
for i in get_tqdm_progressbar(xrange(N_tm // 2 - 1)):
data = overlap[i * N_tm + i + 1:i * N_tm + N_tm // 2].toarray()
maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1)
maxlag[i + 1:, i] = -maxlag[i, i + 1:]
maxoverlap[i, i + 1:] = numpy.max(data, 1)
maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:]
myfile2.close()
def _update_rate_values(self):
"""
Updates the values in sampling points of the following values:
- template width (N_t) in [detection]
- minimal distance between peaks (dist_peaks) in [detection]
- the template shift (template_shift) in [detection]
- the jitter range (jitter_range) in [detection]
- the (savgol_window) in [clustering]
- the (chunk_size) in [data, whitening, fitting]
- the (safety_time) in [clustering, whitening, extracting]
- the (refractory) in [fitting]
"""
if self._N_t is None:
if comm.rank == 0:
print_and_log(
['Changing all values in the param depending on the rate'],
'debug', logger)
spike_width = self.getfloat('detection', 'spike_width')
self._N_t = self.getfloat('detection', 'N_t')
# template width from milisecond to sampling points
self._N_t = int(self.rate * self._N_t * 1e-3)
if numpy.mod(self._N_t, 2) == 0:
self._N_t += 1
self.set('detection', 'N_t', self._N_t)
self.set('detection', 'dist_peaks', self._N_t)
self.set('detection', 'template_shift', (self._N_t - 1) // 2)
self.set('detection', 'spike_width',
int(self.rate * spike_width * 1e-3))
# jitter_range form milisecond sampling points
jitter = self.getfloat('detection', 'jitter_range')
jitter_range = int(self.rate * jitter * 1e-3)
self.set('detection', 'jitter_range', jitter_range)
# savgol from milisecond to sampling points
self._savgol = int(self.rate * 0.5 * 1e-3)
if numpy.mod(self._savgol, 2) == 0:
self._savgol += 1
self.set('clustering', 'savgol_window', self._savgol)
if self.parser._sections['fitting'].has_key('chunk'):
self.parser.set('fitting', 'chunk_size',
self.parser._sections['fitting']['chunk'])
# chunck_size from second to sampling points
for section in ['data', 'whitening', 'fitting']:
chunk = self.parser.getfloat(section, 'chunk_size')
chunk_size = int(chunk * self.rate)
self.set(section, 'chunk_size', chunk_size)
# safety_time from milisecond to sampling points
for section in ['clustering', 'whitening', 'extracting']:
safety_time = self.get(section, 'safety_time')
if safety_time == 'auto':
self.set(section, 'safety_time', self._N_t // 3)
else:
safety_time = int(float(safety_time) * self.rate * 1e-3)
self.set(section, 'safety_time', safety_time)
# refractory from milisecond to sampling points
refractory = self.getfloat('fitting', 'refractory')
self.set('fitting', 'refractory',
int(refractory * self.rate * 1e-3))