def __init__(self, file_name, create_folders=True, **kwargs):
"""
Parameters:
----------
file_name : string
a path containing the *params file.
create_folders : bool
if a folder will be created. If true, output_dir in the data section
of the param file will be created.
Returns:
--------
a CircusParser object.
"""
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.do_folders = create_folders
self.parser = configparser.ConfigParser()
valid_path = check_valid_path(self.file_params)
if not valid_path:
print_and_log(
["Not all nodes can read/write the data file. Check path?"],
'error', logger)
sys.exit(0)
# # First, we remove all tabulations from the parameter file, in order to secure the parser.
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in kwargs.items():
for section in self.__all_sections__:
if self.parser._sections[section].has_key(key):
self.parser._sections[section][key] = value
if self.do_folders and self.parser.get('data', 'output_dir') == '':
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out) and self.do_folders:
os.makedirs(file_out)
self.logfile = file_out + '.log'
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.logfile = f_next + '.log'
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
# read .prb file
try:
self.parser.get('detection', 'radius')
except Exception: # radius == auto by default
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception: # when radius = auto in params file
self.probe = read_probe(self.parser, radius_in_probe=True)
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
else:
self.probe = read_probe(self.parser, radius_in_probe=False)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in self.probe['channel_groups'].keys():
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
"",
"To get more info on a given file format, see",
">> spyking-circus file_format -i",
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in __supported_data_files__.keys()
if not test:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
"",
"To get more info on a given file format, see",
">> spyking-circus file_format -i",
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'clean_artefact'):
for key in ['trig_file', 'trig_windows']:
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', key)))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(
["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', key, myfile)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'ignore_times'):
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file')))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', 'dead_file', myfile)
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'mean-raw'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, mean-raw!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
sys.exit(0)
common_ground = self.parser.get('filtering', 'common_ground')
if common_ground != '':
try:
self.parser.set('filtering', 'common_ground',
str(int(common_ground)))
except Exception:
self.parser.set('filtering', 'common_ground', '-1')
else:
self.parser.set('filtering', 'common_ground', '-1')
common_ground = self.parser.getint('filtering', 'common_ground')
all_electrodes = []
for key in self.probe['channel_groups'].keys():
all_electrodes += self.probe['channel_groups'][key]['channels']
test = (common_ground == -1) or common_ground in all_electrodes
if not test:
if comm.rank == 0:
print_and_log([
"Common ground in filtering section should be a valid electrode"
], 'error', logger)
sys.exit(0)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('fitting', 'ratio_thresh') >
0) and (self.parser.getfloat('fitting', 'ratio_thresh') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["ratio_thresh in [fitting] should be in ]0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
# test = (self.parser.getfloat('clustering', 'ignored_mixtures') >= 0) and (self.parser.getfloat('clustering', 'ignored_mixtures') <= 1)
# if not test:
# if comm.rank == 0:
# print_and_log(["ignored_mixtures in [validating] should be in [0,1]"], 'error', logger)
# sys.exit(0)
test = (self.parser.getfloat('data', 'memory_usage') >
0) and (self.parser.getfloat('data', 'memory_usage') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["memory_usage in [data] should be in ]0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >=
0) and (self.parser.getfloat('merging', 'auto_mode') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be in [0, 1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'noise_limit') >= 0)
if not test:
if comm.rank == 0:
print_and_log(["noise_limit in [merging] should be > 0"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'sparsity_limit') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["sparsity_limit in [merging] should be < 1"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be positive["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('detection', 'smoothing_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["smoothing_factor in [detection] should be positive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering']:
test = self.parser.get('clustering',
'debug_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"debug_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
methods = [
'distance', 'dip', 'folding', 'nd-folding', 'bhatta', 'nd-bhatta'
]
test = self.parser.get('clustering',
'merging_method').lower() in methods
if not test:
if comm.rank == 0:
print_and_log([
"merging_method in [%s] should be in %s" %
(section, str(methods))
], 'error', logger)
sys.exit(0)
if self.parser.get('clustering', 'merging_param').lower() == 'default':
method = self.parser.get('clustering', 'merging_method').lower()
if method == 'dip':
self.parser.set('clustering', 'merging_param', '0.5')
elif method == 'distance':
self.parser.set('clustering', 'merging_param', '3')
elif method in ['folding', 'nd-folding']:
self.parser.set('clustering', 'merging_param', '1e-9')
elif method in ['bhatta', 'nd-bhatta']:
self.parser.set('clustering', 'merging_param', '2')
has_same_elec = self.parser.has_option('clustering', 'sim_same_elec')
has_dip_thresh = self.parser.has_option('clustering', 'dip_threshold')
if has_dip_thresh:
dip_threshold = self.parser.getfloat('clustering', 'dip_threshold')
if has_dip_thresh and dip_threshold > 0:
if comm.rank == 0:
print_and_log([
"dip_threshold in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"By default a nd-bhatta merging method with param 3 is assumed"
], 'info', logger)
self.parser.set('clustering', 'merging_param', str(3))
self.parser.set('clustering', 'merging_method', 'distance')
elif has_same_elec:
sim_same_elec = self.parser.get('clustering', 'sim_same_elec')
if comm.rank == 0:
print_and_log([
"sim_same_elec in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"Meanwhile a distance merging method with param %s is assumed"
% sim_same_elec
], 'info', logger)
self.parser.set('clustering', 'merging_param', sim_same_elec)
self.parser.set('clustering', 'merging_method', 'distance')
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = map(float, dispersion)
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
if self.parser.getboolean('detection', 'hanning'):
if comm.rank == 0:
print_and_log(["Hanning filtering is activated"], 'debug',
logger)
def __init__(self, file_name, **kwargs):
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.logfile = f_next + '.log'
self.parser = configparser.ConfigParser()
## First, we remove all tabulations from the parameter file, in order
## to secure the parser
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in list(kwargs.items()):
for section in self.__all_sections__:
if key in self.parser._sections[section]:
self.parser._sections[section][key] = value
self.probe = read_probe(self.parser)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in list(self.probe['channel_groups'].keys()):
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in list(
__supported_data_files__.keys())
if not test:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'trig_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
self.parser.set(
'triggers', 'trig_windows',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers',
'trig_windows'))))
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'dead_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file'))))
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out):
os.makedirs(file_out)
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'noise_thr') >=
0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >= 0)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be positive"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be postiive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = list(map(float, dispersion))
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
def __init__(self, file_name, **kwargs):
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.logfile = f_next + '.log'
self.parser = configparser.ConfigParser()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" %self.file_params], 'error', logger)
sys.exit(1)
if comm.rank == 0:
print_and_log(['Creating a Circus Parser for datafile %s' %self.file_name], 'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in kwargs.items():
for section in self.__all_sections__:
if self.parser._sections[section].has_key(key):
self.parser._sections[section][key] = value
self.probe = read_probe(self.parser)
N_e = 0
for key in self.probe['channel_groups'].keys():
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log(['The number of analyzed channels is higher than the number of recorded channels'], 'error', logger)
sys.exit(1)
to_write = ["You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", '']
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]
to_write += ['', "To get more info on a given file format, see",
">> spyking-circus file_format -i"]
print_and_log(to_write, 'error', logger)
sys.exit(1)
test = self.file_format.lower() in __supported_data_files__.keys()
if not test:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += ['-- %s -- %s' %(f, __supported_data_files__[f].extension)]
to_write += ['', "To get more info on a given file format, see",
">> spyking-circus file_format -i"]
print_and_log(to_write, 'error', logger)
sys.exit(1)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers', 'trig_file') == '') or (self.parser.get('triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log(["trig_file and trig_windows must be specified in [triggers]"], 'error', logger)
sys.exit(1)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(["trig_unit in [triggers] should be in %s" %str(units)], 'error', logger)
sys.exit(1)
else:
self.parser.set('triggers', 'trig_in_ms', str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
self.parser.set('triggers', 'trig_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
self.parser.set('triggers', 'trig_windows', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'trig_windows'))))
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(["dead_unit in [triggers] should be in %s" %str(units)], 'error', logger)
sys.exit(1)
else:
self.parser.set('triggers', 'dead_in_ms', str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
self.parser.set('triggers', 'dead_file', os.path.abspath(os.path.expanduser(self.parser.get('triggers', 'dead_file'))))
test = (self.parser.get('clustering', 'extraction').lower() in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log(["Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"], 'error', logger)
sys.exit(1)
test = (self.parser.get('detection', 'peaks').lower() in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log(["Only 3 detection modes for peaks in [detection]: negative, positive, both"], 'error', logger)
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
self.parser.set('data', 'data_file_no_overwrite', os.path.join(file_path, os.path.basename(f_next) + '_all_sc.dat'))
file_out = os.path.join(f_next, os.path.basename(f_next))
self.parser.set('data', 'file_out', file_out) # Output file without suffix
self.parser.set('data', 'file_out_suff', file_out + self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext', f_next) # Data file (assuming .filtered at the end)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') > 0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["nb_elts in [%s] should be in [0,1]" %section], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'noise_thr') >= 0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('validating', 'test_size') > 0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'cc_merge') >= 0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('clustering', 'nclus_min') >= 0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["], 'error', logger)
sys.exit(1)
test = (self.parser.getfloat('merging', 'auto_mode') >= 0) and (self.parser.getfloat('merging', 'auto_mode') < 1)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be in [0,1["], 'error', logger)
sys.exit(1)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering', 'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log(["make_plots in [%s] should be in %s" %(section, str(fileformats))], 'error', logger)
sys.exit(1)
dispersion = self.parser.get('clustering', 'dispersion').replace('(', '').replace(')', '').split(',')
dispersion = map(float, dispersion)
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log(["min and max dispersions in [clustering] should be positive"], 'error', logger)
sys.exit(1)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting', 'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log(["export_pcs in [converting] should be in %s" %str(pcs_export)], 'error', logger)
sys.exit(1)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
