def slice_clusters(params, result, to_remove=[], to_merge=[], extension='', light=False):
import h5py, shutil
file_out_suff = params.get('data', 'file_out_suff')
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
if comm.rank == 0:
print_and_log(['Node 0 is slicing clusters'], 'debug', logger)
if to_merge != []:
for count in xrange(len(to_merge)):
remove = to_merge[count][1]
to_remove += [remove]
all_elements = [[] for i in xrange(N_e)]
for target in numpy.unique(to_remove):
elec = result['electrodes'][target]
nic = target - numpy.where(result['electrodes'] == elec)[0][0]
mask = result['clusters_' + str(elec)] > -1
tmp = numpy.unique(result['clusters_' + str(elec)][mask])
all_elements[elec] += list(numpy.where(result['clusters_' + str(elec)] == tmp[nic])[0])
for elec in xrange(N_e):
if not light:
result['data_' + str(elec)] = numpy.delete(result['data_' + str(elec)], all_elements[elec], axis=0)
result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec])
result['times_' + str(elec)] = numpy.delete(result['times_' + str(elec)], all_elements[elec])
result['peaks_' + str(elec)] = numpy.delete(result['peaks_' + str(elec)], all_elements[elec])
else:
result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], all_elements[elec])
myfile = h5py.File(file_out_suff + '.clusters.hdf5', 'r', libver='earliest')
data = myfile.get('data_' + str(elec))[:]
result['data_' + str(elec)] = numpy.delete(data, all_elements[elec], axis=0)
data = myfile.get('times_' + str(elec))[:]
result['times_' + str(elec)] = numpy.delete(data, all_elements[elec])
data = myfile.get('peaks_' + str(elec))[:]
result['peaks_' + str(elec)] = numpy.delete(data, all_elements[elec])
myfile.close()
result['electrodes'] = numpy.delete(result['electrodes'], numpy.unique(to_remove))
cfile = h5py.File(file_out_suff + '.clusters-new.hdf5', 'w', libver='earliest')
to_write = ['data_', 'clusters_', 'times_', 'peaks_']
for ielec in xrange(N_e):
write_datasets(cfile, to_write, result, ielec)
write_datasets(cfile, ['electrodes'], result)
cfile.close()
if os.path.exists(file_out_suff + '.clusters%s.hdf5' %extension):
os.remove(file_out_suff + '.clusters%s.hdf5' %extension)
shutil.move(file_out_suff + '.clusters-new.hdf5', file_out_suff + '.clusters%s.hdf5' %extension)
comm.Barrier()
def delete_mixtures(params, nb_cpu, nb_gpu, use_gpu):
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
cc_merge = params.getfloat('clustering', 'cc_mixtures')
mixtures = []
to_remove = []
filename = params.get('data', 'file_out_suff') + '.overlap-mixtures.hdf5'
norm_templates = load_data(params, 'norm-templates')
best_elec = load_data(params, 'electrodes')
limits = load_data(params, 'limits')
nodes, edges = get_nodes_and_edges(params)
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
overlap = get_overlaps(params,
extension='-mixtures',
erase=True,
normalize=False,
maxoverlap=False,
verbose=False,
half=True,
use_gpu=use_gpu,
nb_cpu=nb_cpu,
nb_gpu=nb_gpu)
overlap.close()
SHARED_MEMORY = get_shared_memory_flag(params)
if SHARED_MEMORY:
c_overs = load_data_memshared(params,
'overlaps',
extension='-mixtures',
use_gpu=use_gpu,
nb_cpu=nb_cpu,
nb_gpu=nb_gpu)
else:
c_overs = load_data(params, 'overlaps', extension='-mixtures')
if SHARED_MEMORY:
templates = load_data_memshared(params, 'templates', normalize=False)
else:
templates = load_data(params, 'templates')
x, N_tm = templates.shape
nb_temp = int(N_tm // 2)
merged = [nb_temp, 0]
overlap_0 = numpy.zeros(nb_temp, dtype=numpy.float32)
distances = numpy.zeros((nb_temp, nb_temp), dtype=numpy.int32)
for i in xrange(nb_temp - 1):
data = c_overs[i].toarray()
distances[i, i + 1:] = numpy.argmax(data[i + 1:, :], 1)
distances[i + 1:, i] = distances[i, i + 1:]
overlap_0[i] = data[i, N_t]
all_temp = numpy.arange(comm.rank, nb_temp, comm.size)
sorted_temp = numpy.argsort(
norm_templates[:nb_temp])[::-1][comm.rank::comm.size]
M = numpy.zeros((2, 2), dtype=numpy.float32)
V = numpy.zeros((2, 1), dtype=numpy.float32)
to_explore = xrange(comm.rank, len(sorted_temp), comm.size)
if comm.rank == 0:
to_explore = get_tqdm_progressbar(to_explore)
for count, k in enumerate(to_explore):
k = sorted_temp[k]
electrodes = numpy.take(inv_nodes, edges[nodes[best_elec[k]]])
overlap_k = c_overs[k]
is_in_area = numpy.in1d(best_elec, electrodes)
all_idx = numpy.arange(len(best_elec))[is_in_area]
been_found = False
t_k = None
for i in all_idx:
t_i = None
if not been_found:
overlap_i = c_overs[i]
M[0, 0] = overlap_0[i]
V[0, 0] = overlap_k[i, distances[k, i]]
for j in all_idx[i + 1:]:
t_j = None
M[1, 1] = overlap_0[j]
M[1, 0] = overlap_i[j, distances[k, i] - distances[k, j]]
M[0, 1] = M[1, 0]
V[1, 0] = overlap_k[j, distances[k, j]]
try:
[a1, a2] = numpy.dot(scipy.linalg.inv(M), V)
except Exception:
[a1, a2] = [0, 0]
a1_lim = limits[i]
a2_lim = limits[j]
is_a1 = (a1_lim[0] <= a1) and (a1 <= a1_lim[1])
is_a2 = (a2_lim[0] <= a2) and (a2 <= a2_lim[1])
if is_a1 and is_a2:
if t_k is None:
t_k = templates[:, k].toarray().ravel()
if t_i is None:
t_i = templates[:, i].toarray().ravel()
if t_j is None:
t_j = templates[:, j].toarray().ravel()
new_template = (a1 * t_i + a2 * t_j)
similarity = numpy.corrcoef(t_k, new_template)[0, 1]
local_overlap = numpy.corrcoef(t_i, t_j)[0, 1]
if similarity > cc_merge and local_overlap < cc_merge:
if k not in mixtures:
mixtures += [k]
been_found = True
#print "Template", k, 'is sum of (%d, %g) and (%d,%g)' %(i, a1, j, a2)
break
sys.stderr.flush()
#print mixtures
to_remove = numpy.unique(numpy.array(mixtures, dtype=numpy.int32))
to_remove = all_gather_array(to_remove, comm, 0, dtype='int32')
if len(to_remove) > 0 and comm.rank == 0:
result = load_data(params, 'clusters')
slice_templates(params, to_remove)
slice_clusters(params, result, to_remove=to_remove)
comm.Barrier()
del c_overs
if comm.rank == 0:
os.remove(filename)
return [nb_temp, len(to_remove)]
def merging_cc(params, nb_cpu, nb_gpu, use_gpu):
def remove(result, distances, cc_merge):
do_merge = True
to_merge = numpy.zeros((0, 2), dtype=numpy.int32)
g_idx = range(len(distances))
while do_merge:
dmax = distances.max()
idx = numpy.where(distances == dmax)
one_merge = [idx[0][0], idx[1][0]]
do_merge = dmax >= cc_merge
if do_merge:
elec_ic1 = result['electrodes'][one_merge[0]]
elec_ic2 = result['electrodes'][one_merge[1]]
nic1 = one_merge[0] - numpy.where(
result['electrodes'] == elec_ic1)[0][0]
nic2 = one_merge[1] - numpy.where(
result['electrodes'] == elec_ic2)[0][0]
mask1 = result['clusters_' + str(elec_ic1)] > -1
mask2 = result['clusters_' + str(elec_ic2)] > -1
tmp1 = numpy.unique(result['clusters_' + str(elec_ic1)][mask1])
tmp2 = numpy.unique(result['clusters_' + str(elec_ic2)][mask2])
elements1 = numpy.where(result['clusters_' +
str(elec_ic1)] == tmp1[nic1])[0]
elements2 = numpy.where(result['clusters_' +
str(elec_ic2)] == tmp2[nic2])[0]
if len(elements1) > len(elements2):
to_remove = one_merge[1]
to_keep = one_merge[0]
elec = elec_ic2
elements = elements2
else:
to_remove = one_merge[0]
to_keep = one_merge[1]
elec = elec_ic1
elements = elements1
result['data_' + str(elec)] = numpy.delete(result['data_' +
str(elec)],
elements,
axis=0)
result['clusters_' + str(elec)] = numpy.delete(
result['clusters_' + str(elec)], elements)
result['times_' + str(elec)] = numpy.delete(
result['times_' + str(elec)], elements)
result['peaks_' + str(elec)] = numpy.delete(
result['peaks_' + str(elec)], elements)
result['electrodes'] = numpy.delete(result['electrodes'],
to_remove)
distances = numpy.delete(distances, to_remove, axis=0)
distances = numpy.delete(distances, to_remove, axis=1)
to_merge = numpy.vstack(
(to_merge, numpy.array([g_idx[to_keep],
g_idx[to_remove]])))
g_idx.pop(to_remove)
return to_merge, result
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
blosc_compress = params.getboolean('data', 'blosc_compress')
N_tm = load_data(params, 'nb_templates')
nb_temp = int(N_tm // 2)
to_merge = []
cc_merge = params.getfloat('clustering', 'cc_merge')
norm = N_e * N_t
result = []
overlap = get_overlaps(params,
extension='-merging',
erase=True,
normalize=True,
maxoverlap=False,
verbose=False,
half=True,
use_gpu=use_gpu,
nb_cpu=nb_cpu,
nb_gpu=nb_gpu)
overlap.close()
filename = params.get('data', 'file_out_suff') + '.overlap-merging.hdf5'
SHARED_MEMORY = get_shared_memory_flag(params)
if not SHARED_MEMORY:
over_x, over_y, over_data, over_shape = load_data(params,
'overlaps-raw',
extension='-merging')
else:
over_x, over_y, over_data, over_shape = load_data_memshared(
params,
'overlaps-raw',
extension='-merging',
use_gpu=use_gpu,
nb_cpu=nb_cpu,
nb_gpu=nb_gpu)
#sub_comm, is_local = get_local_ring(True)
#if is_local:
distances = numpy.zeros((nb_temp, nb_temp), dtype=numpy.float32)
to_explore = numpy.arange(nb_temp - 1)[comm.rank::comm.size]
for i in to_explore:
idx = numpy.where((over_x >= i * nb_temp + i + 1)
& (over_x < ((i + 1) * nb_temp)))[0]
local_x = over_x[idx] - (i * nb_temp + i + 1)
data = numpy.zeros((nb_temp - (i + 1), over_shape[1]),
dtype=numpy.float32)
data[local_x, over_y[idx]] = over_data[idx]
distances[i, i + 1:] = numpy.max(data, 1) / norm
distances[i + 1:, i] = distances[i, i + 1:]
#Now we need to sync everything across nodes
distances = gather_array(distances,
comm,
0,
1,
'float32',
compress=blosc_compress)
if comm.rank == 0:
distances = distances.reshape(comm.size, nb_temp, nb_temp)
distances = numpy.sum(distances, 0)
#sub_comm.Barrier()
#sub_comm.Free()
if comm.rank == 0:
result = load_data(params, 'clusters')
to_merge, result = remove(result, distances, cc_merge)
to_merge = numpy.array(to_merge)
to_merge = comm.bcast(to_merge, root=0)
if len(to_merge) > 0:
slice_templates(params, to_merge=to_merge)
slice_clusters(params, result)
comm.Barrier()
del result, over_x, over_y, over_data
if comm.rank == 0:
os.remove(filename)
return [nb_temp, len(to_merge)]
def __init__(self, file_name, **kwargs):
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.logfile = f_next + '.log'
self.parser = configparser.ConfigParser()
## First, we remove all tabulations from the parameter file, in order
## to secure the parser
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in list(kwargs.items()):
for section in self.__all_sections__:
if key in self.parser._sections[section]:
self.parser._sections[section][key] = value
self.probe = read_probe(self.parser)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in list(self.probe['channel_groups'].keys()):
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in list(
__supported_data_files__.keys())
if not test:
if comm.rank == 0:
for f in list(__supported_data_files__.keys()):
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
'', "To get more info on a given file format, see",
">> spyking-circus file_format -i"
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'trig_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'trig_file'))))
self.parser.set(
'triggers', 'trig_windows',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers',
'trig_windows'))))
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
self.parser.set(
'triggers', 'dead_file',
os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file'))))
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'median-pca', 'mean-raw', 'mean-pca'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, median-pca, mean-raw or mean-pca!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out):
os.makedirs(file_out)
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [clustering] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'noise_thr') >=
0) and (self.parser.getfloat('clustering', 'noise_thr') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["noise_thr in [clustering] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'nclus_min') >=
0) and (self.parser.getfloat('clustering', 'nclus_min') < 1)
if not test:
if comm.rank == 0:
print_and_log(["nclus_min in [validating] should be in [0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >= 0)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be positive"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be postiive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = list(map(float, dispersion))
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
def delete_mixtures(params, nb_cpu, nb_gpu, use_gpu):
templates = load_data(params, 'templates')
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
cc_merge = params.getfloat('clustering', 'cc_merge')
x, N_tm = templates.shape
nb_temp = N_tm//2
merged = [nb_temp, 0]
mixtures = []
to_remove = []
overlap = get_overlaps(params, extension='-mixtures', erase=True, normalize=False, maxoverlap=False, verbose=False, half=True, use_gpu=use_gpu, nb_cpu=nb_cpu, nb_gpu=nb_gpu)
filename = params.get('data', 'file_out_suff') + '.overlap-mixtures.hdf5'
result = []
norm_templates = load_data(params, 'norm-templates')
templates = load_data(params, 'templates')
result = load_data(params, 'clusters')
best_elec = load_data(params, 'electrodes')
limits = load_data(params, 'limits')
nodes, edges = get_nodes_and_edges(params)
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
distances = numpy.zeros((nb_temp, nb_temp), dtype=numpy.float32)
over_x = overlap.get('over_x')[:]
over_y = overlap.get('over_y')[:]
over_data = overlap.get('over_data')[:]
over_shape = overlap.get('over_shape')[:]
overlap.close()
overlap = scipy.sparse.csr_matrix((over_data, (over_x, over_y)), shape=over_shape)
for i in xrange(nb_temp-1):
distances[i, i+1:] = numpy.argmax(overlap[i*nb_temp+i+1:(i+1)*nb_temp].toarray(), 1)
distances[i+1:, i] = distances[i, i+1:]
all_temp = numpy.arange(comm.rank, nb_temp, comm.size)
overlap_0 = overlap[:, N_t].toarray().reshape(nb_temp, nb_temp)
sorted_temp = numpy.argsort(norm_templates[:nb_temp])[::-1][comm.rank::comm.size]
M = numpy.zeros((2, 2), dtype=numpy.float32)
V = numpy.zeros((2, 1), dtype=numpy.float32)
to_explore = xrange(comm.rank, len(sorted_temp), comm.size)
if comm.rank == 0:
to_explore = get_tqdm_progressbar(to_explore)
for count, k in enumerate(to_explore):
k = sorted_temp[k]
electrodes = numpy.take(inv_nodes, edges[nodes[best_elec[k]]])
overlap_k = overlap[k*nb_temp:(k+1)*nb_temp].tolil()
is_in_area = numpy.in1d(best_elec, electrodes)
all_idx = numpy.arange(len(best_elec))[is_in_area]
been_found = False
for i in all_idx:
if not been_found:
overlap_i = overlap[i*nb_temp:(i+1)*nb_temp].tolil()
M[0, 0] = overlap_0[i, i]
V[0, 0] = overlap_k[i, distances[k, i]]
for j in all_idx[i+1:]:
M[1, 1] = overlap_0[j, j]
M[1, 0] = overlap_i[j, distances[k, i] - distances[k, j]]
M[0, 1] = M[1, 0]
V[1, 0] = overlap_k[j, distances[k, j]]
try:
[a1, a2] = numpy.dot(scipy.linalg.inv(M), V)
except Exception:
[a1, a2] = [0, 0]
a1_lim = limits[i]
a2_lim = limits[j]
is_a1 = (a1_lim[0] <= a1) and (a1 <= a1_lim[1])
is_a2 = (a2_lim[0] <= a2) and (a2 <= a2_lim[1])
if is_a1 and is_a2:
new_template = (a1*templates[:, i].toarray() + a2*templates[:, j].toarray()).ravel()
similarity = numpy.corrcoef(templates[:, k].toarray().ravel(), new_template)[0, 1]
if similarity > cc_merge:
if k not in mixtures:
mixtures += [k]
been_found = True
break
#print "Template", k, 'is sum of (%d, %g) and (%d,%g)' %(i, a1, j, a2)
#print mixtures
to_remove = numpy.unique(numpy.array(mixtures, dtype=numpy.int32))
to_remove = all_gather_array(to_remove, comm, 0, dtype='int32')
if len(to_remove) > 0:
slice_templates(params, to_remove)
slice_clusters(params, result, to_remove=to_remove)
comm.Barrier()
if comm.rank == 0:
os.remove(filename)
return [nb_temp, len(to_remove)]
def merging_cc(params, nb_cpu, nb_gpu, use_gpu):
def remove(result, distances, cc_merge):
do_merge = True
to_merge = numpy.zeros((0, 2), dtype=numpy.int32)
g_idx = range(len(distances))
while do_merge:
dmax = distances.max()
idx = numpy.where(distances == dmax)
one_merge = [idx[0][0], idx[1][0]]
do_merge = dmax >= cc_merge
if do_merge:
elec_ic1 = result['electrodes'][one_merge[0]]
elec_ic2 = result['electrodes'][one_merge[1]]
nic1 = one_merge[0] - numpy.where(result['electrodes'] == elec_ic1)[0][0]
nic2 = one_merge[1] - numpy.where(result['electrodes'] == elec_ic2)[0][0]
mask1 = result['clusters_' + str(elec_ic1)] > -1
mask2 = result['clusters_' + str(elec_ic2)] > -1
tmp1 = numpy.unique(result['clusters_' + str(elec_ic1)][mask1])
tmp2 = numpy.unique(result['clusters_' + str(elec_ic2)][mask2])
elements1 = numpy.where(result['clusters_' + str(elec_ic1)] == tmp1[nic1])[0]
elements2 = numpy.where(result['clusters_' + str(elec_ic2)] == tmp2[nic2])[0]
if len(elements1) > len(elements2):
to_remove = one_merge[1]
to_keep = one_merge[0]
elec = elec_ic2
elements = elements2
else:
to_remove = one_merge[0]
to_keep = one_merge[1]
elec = elec_ic1
elements = elements1
result['data_' + str(elec)] = numpy.delete(result['data_' + str(elec)], elements, axis=0)
result['clusters_' + str(elec)] = numpy.delete(result['clusters_' + str(elec)], elements)
result['times_' + str(elec)] = numpy.delete(result['times_' + str(elec)], elements)
result['peaks_' + str(elec)] = numpy.delete(result['peaks_' + str(elec)], elements)
result['electrodes'] = numpy.delete(result['electrodes'], to_remove)
distances = numpy.delete(distances, to_remove, axis=0)
distances = numpy.delete(distances, to_remove, axis=1)
to_merge = numpy.vstack((to_merge, numpy.array([g_idx[to_keep], g_idx[to_remove]])))
g_idx.pop(to_remove)
return to_merge, result
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
templates = load_data(params, 'templates')
x, N_tm = templates.shape
nb_temp = N_tm//2
to_merge = []
cc_merge = params.getfloat('clustering', 'cc_merge')
result = []
overlap = get_overlaps(params, extension='-merging', erase=True, normalize=True, maxoverlap=False, verbose=False, half=True, use_gpu=use_gpu, nb_cpu=nb_cpu, nb_gpu=nb_gpu)
filename = params.get('data', 'file_out_suff') + '.overlap-merging.hdf5'
if comm.rank > 0:
overlap.close()
else:
over_x = overlap.get('over_x')[:]
over_y = overlap.get('over_y')[:]
over_data = overlap.get('over_data')[:]
over_shape = overlap.get('over_shape')[:]
overlap.close()
overlap = scipy.sparse.csr_matrix((over_data, (over_x, over_y)), shape=over_shape)
result = load_data(params, 'clusters')
distances = numpy.zeros((nb_temp, nb_temp), dtype=numpy.float32)
for i in xrange(nb_temp-1):
distances[i, i+1:] = numpy.max(overlap[i*nb_temp+i+1:(i+1)*nb_temp].toarray(), 1)
distances[i+1:, i] = distances[i, i+1:]
distances /= (N_e*N_t)
to_merge, result = remove(result, distances, cc_merge)
to_merge = numpy.array(to_merge)
to_merge = comm.bcast(to_merge, root=0)
if len(to_merge) > 0:
slice_templates(params, to_merge=to_merge)
slice_clusters(params, result)
comm.Barrier()
if comm.rank == 0:
os.remove(filename)
return [nb_temp, len(to_merge)]
def slice_templates(params, to_remove=[], to_merge=[], extension=''):
import shutil, h5py
file_out_suff = params.get('data', 'file_out_suff')
data_file = params.data_file
N_e = params.getint('data', 'N_e')
N_total = params.nb_channels
N_t = params.getint('detection', 'N_t')
template_shift = params.getint('detection', 'template_shift')
if comm.rank == 0:
print_and_log(['Node 0 is slicing templates'], 'debug', logger)
old_templates = load_data(params, 'templates')
old_limits = load_data(params, 'limits')
x, N_tm = old_templates.shape
norm_templates = load_data(params, 'norm-templates')
if to_merge != []:
for count in xrange(len(to_merge)):
remove = to_merge[count][1]
to_remove += [remove]
all_templates = set(numpy.arange(N_tm//2))
to_keep = numpy.array(list(all_templates.difference(to_remove)))
positions = numpy.arange(len(to_keep))
local_keep = to_keep[positions]
templates = scipy.sparse.lil_matrix((N_e*N_t, 2*len(to_keep)), dtype=numpy.float32)
hfile = h5py.File(file_out_suff + '.templates-new.hdf5', 'w', libver='earliest')
norms = hfile.create_dataset('norms', shape=(2*len(to_keep), ), dtype=numpy.float32, chunks=True)
limits = hfile.create_dataset('limits', shape=(len(to_keep), 2), dtype=numpy.float32, chunks=True)
for count, keep in zip(positions, local_keep):
templates[:, count] = old_templates[:, keep]
templates[:, count + len(to_keep)] = old_templates[:, keep + N_tm//2]
norms[count] = norm_templates[keep]
norms[count + len(to_keep)] = norm_templates[keep + N_tm//2]
if to_merge == []:
new_limits = old_limits[keep]
else:
subset = numpy.where(to_merge[:, 0] == keep)[0]
if len(subset) > 0:
idx = numpy.unique(to_merge[subset].flatten())
ratios = norm_templates[idx]/norm_templates[keep]
new_limits = [numpy.min(ratios*old_limits[idx][:, 0]), numpy.max(ratios*old_limits[idx][:, 1])]
else:
new_limits = old_limits[keep]
limits[count] = new_limits
templates = templates.tocoo()
hfile.create_dataset('temp_x', data=templates.row)
hfile.create_dataset('temp_y', data=templates.col)
hfile.create_dataset('temp_data', data=templates.data)
hfile.create_dataset('temp_shape', data=numpy.array([N_e, N_t, 2*len(to_keep)], dtype=numpy.int32))
hfile.close()
if os.path.exists(file_out_suff + '.templates%s.hdf5' %extension):
os.remove(file_out_suff + '.templates%s.hdf5' %extension)
shutil.move(file_out_suff + '.templates-new.hdf5', file_out_suff + '.templates%s.hdf5' %extension)
comm.Barrier()
def __init__(self, file_name, create_folders=True, **kwargs):
"""
Parameters:
----------
file_name : string
a path containing the *params file.
create_folders : bool
if a folder will be created. If true, output_dir in the data section
of the param file will be created.
Returns:
--------
a CircusParser object.
"""
self.file_name = os.path.abspath(file_name)
f_next, extension = os.path.splitext(self.file_name)
file_path = os.path.dirname(self.file_name)
self.file_params = f_next + '.params'
self.do_folders = create_folders
self.parser = configparser.ConfigParser()
valid_path = check_valid_path(self.file_params)
if not valid_path:
print_and_log(
["Not all nodes can read/write the data file. Check path?"],
'error', logger)
sys.exit(0)
# # First, we remove all tabulations from the parameter file, in order to secure the parser.
if comm.rank == 0:
myfile = open(self.file_params, 'r')
lines = myfile.readlines()
myfile.close()
myfile = open(self.file_params, 'w')
for l in lines:
myfile.write(l.replace('\t', ''))
myfile.close()
comm.Barrier()
self._N_t = None
if not os.path.exists(self.file_params):
if comm.rank == 0:
print_and_log(["%s does not exist" % self.file_params],
'error', logger)
sys.exit(0)
if comm.rank == 0:
print_and_log(
['Creating a Circus Parser for datafile %s' % self.file_name],
'debug', logger)
self.parser.read(self.file_params)
for section in self.__all_sections__:
if self.parser.has_section(section):
for (key, value) in self.parser.items(section):
self.parser.set(section, key, value.split('#')[0].rstrip())
else:
self.parser.add_section(section)
for item in self.__default_values__ + self.__extra_values__:
section, name, val_type, value = item
try:
if val_type is 'bool':
self.parser.getboolean(section, name)
elif val_type is 'int':
self.parser.getint(section, name)
elif val_type is 'float':
self.parser.getfloat(section, name)
elif val_type is 'string':
self.parser.get(section, name)
except Exception:
self.parser.set(section, name, value)
for key, value in kwargs.items():
for section in self.__all_sections__:
if self.parser._sections[section].has_key(key):
self.parser._sections[section][key] = value
if self.do_folders and self.parser.get('data', 'output_dir') == '':
try:
os.makedirs(f_next)
except Exception:
pass
self.parser.set('data', 'data_file', self.file_name)
if self.parser.get('data', 'output_dir') != '':
path = os.path.abspath(
os.path.expanduser(self.parser.get('data', 'output_dir')))
self.parser.set('data', 'output_dir', path)
file_out = os.path.join(path, os.path.basename(f_next))
if not os.path.exists(file_out) and self.do_folders:
os.makedirs(file_out)
self.logfile = file_out + '.log'
else:
file_out = os.path.join(f_next, os.path.basename(f_next))
self.logfile = f_next + '.log'
self.parser.set('data', 'data_file_no_overwrite',
file_out + '_all_sc.dat')
self.parser.set('data', 'file_out',
file_out) # Output file without suffix
self.parser.set(
'data', 'file_out_suff', file_out +
self.parser.get('data', 'suffix')) # Output file with suffix
self.parser.set('data', 'data_file_noext',
f_next) # Data file (assuming .filtered at the end)
# read .prb file
try:
self.parser.get('detection', 'radius')
except Exception: # radius == auto by default
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception: # when radius = auto in params file
self.probe = read_probe(self.parser, radius_in_probe=True)
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
else:
self.probe = read_probe(self.parser, radius_in_probe=False)
dead_channels = self.parser.get('detection', 'dead_channels')
if dead_channels != '':
dead_channels = parse_dead_channels(dead_channels)
if comm.rank == 0:
print_and_log(
["Removing dead channels %s" % str(dead_channels)],
'debug', logger)
for key in dead_channels.keys():
if key in self.probe["channel_groups"].keys():
for channel in dead_channels[key]:
n_before = len(
self.probe["channel_groups"][key]['channels'])
self.probe["channel_groups"][key]['channels'] = list(
set(self.probe["channel_groups"][key]
['channels']).difference(dead_channels[key]))
n_after = len(
self.probe["channel_groups"][key]['channels'])
else:
if comm.rank == 0:
print_and_log([
"Probe has no group named %s for dead channels" %
key
], 'debug', logger)
N_e = 0
for key in self.probe['channel_groups'].keys():
N_e += len(self.probe['channel_groups'][key]['channels'])
self.set('data', 'N_e', str(N_e))
self.set('data', 'N_total', str(self.probe['total_nb_channels']))
self.set('data', 'nb_channels', str(self.probe['total_nb_channels']))
self.nb_channels = self.probe['total_nb_channels']
if N_e > self.nb_channels:
if comm.rank == 0:
print_and_log([
'The number of analyzed channels is higher than the number of recorded channels'
], 'error', logger)
sys.exit(0)
if N_e == 1 and self.parser.getboolean('filtering', 'remove_median'):
if comm.rank == 0:
print_and_log([
"With 1 channel, remove_median in [filtering] is not possible"
], 'error', logger)
sys.exit(0)
to_write = [
"You must specify explicitly the file format in the config file",
"Please have a look to the documentation and add a file_format",
"parameter in the [data] section. Valid files formats can be:", ''
]
try:
self.file_format = self.parser.get('data', 'file_format')
except Exception:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
"",
"To get more info on a given file format, see",
">> spyking-circus file_format -i",
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
test = self.file_format.lower() in __supported_data_files__.keys()
if not test:
if comm.rank == 0:
for f in __supported_data_files__.keys():
to_write += [
'-- %s -- %s' %
(f, __supported_data_files__[f].extension)
]
to_write += [
"",
"To get more info on a given file format, see",
">> spyking-circus file_format -i",
]
print_and_log(to_write, 'error', logger)
sys.exit(0)
try:
self.parser.get('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius', 'auto')
try:
self.parser.getint('detection', 'radius')
except Exception:
self.parser.set('detection', 'radius',
str(int(self.probe['radius'])))
if self.parser.getboolean('triggers', 'clean_artefact'):
if (self.parser.get('triggers',
'trig_file') == '') or (self.parser.get(
'triggers', 'trig_windows') == ''):
if comm.rank == 0:
print_and_log([
"trig_file and trig_windows must be specified in [triggers]"
], 'error', logger)
sys.exit(0)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'trig_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["trig_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'trig_in_ms',
str(self.parser.get('triggers', 'trig_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'clean_artefact'):
for key in ['trig_file', 'trig_windows']:
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', key)))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(
["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', key, myfile)
units = ['ms', 'timestep']
test = self.parser.get('triggers', 'dead_unit').lower() in units
if not test:
if comm.rank == 0:
print_and_log(
["dead_unit in [triggers] should be in %s" % str(units)],
'error', logger)
sys.exit(0)
else:
self.parser.set(
'triggers', 'dead_in_ms',
str(self.parser.get('triggers', 'dead_unit').lower() == 'ms'))
if self.parser.getboolean('triggers', 'ignore_times'):
myfile = os.path.abspath(
os.path.expanduser(self.parser.get('triggers', 'dead_file')))
if not os.path.exists(myfile):
if comm.rank == 0:
print_and_log(["File %s can not be found" % str(myfile)],
'error', logger)
sys.exit(0)
self.parser.set('triggers', 'dead_file', myfile)
test = (self.parser.get('clustering', 'extraction').lower()
in ['median-raw', 'mean-raw'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 4 extraction modes in [clustering]: median-raw, mean-raw!"
], 'error', logger)
sys.exit(0)
test = (self.parser.get('detection', 'peaks').lower()
in ['negative', 'positive', 'both'])
if not test:
if comm.rank == 0:
print_and_log([
"Only 3 detection modes for peaks in [detection]: negative, positive, both"
], 'error', logger)
sys.exit(0)
common_ground = self.parser.get('filtering', 'common_ground')
if common_ground != '':
try:
self.parser.set('filtering', 'common_ground',
str(int(common_ground)))
except Exception:
self.parser.set('filtering', 'common_ground', '-1')
else:
self.parser.set('filtering', 'common_ground', '-1')
common_ground = self.parser.getint('filtering', 'common_ground')
all_electrodes = []
for key in self.probe['channel_groups'].keys():
all_electrodes += self.probe['channel_groups'][key]['channels']
test = (common_ground == -1) or common_ground in all_electrodes
if not test:
if comm.rank == 0:
print_and_log([
"Common ground in filtering section should be a valid electrode"
], 'error', logger)
sys.exit(0)
is_cluster = check_if_cluster()
self.parser.set('data', 'is_cluster', str(is_cluster))
if is_cluster:
print_and_log([
"Cluster detected, so using local /tmp folders and blosc compression"
], 'debug', logger)
self.parser.set('data', 'global_tmp', 'False')
self.parser.set('data', 'blosc_compress', 'True')
else:
print_and_log(
["Cluster not detected, so using global /tmp folder"], 'debug',
logger)
for section in ['whitening', 'clustering']:
test = (self.parser.getfloat(section, 'nb_elts') >
0) and (self.parser.getfloat(section, 'nb_elts') <= 1)
if not test:
if comm.rank == 0:
print_and_log(
["nb_elts in [%s] should be in [0,1]" % section],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('validating', 'test_size') >
0) and (self.parser.getfloat('validating', 'test_size') < 1)
if not test:
if comm.rank == 0:
print_and_log(["test_size in [validating] should be in ]0,1["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('fitting', 'ratio_thresh') >
0) and (self.parser.getfloat('fitting', 'ratio_thresh') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["ratio_thresh in [fitting] should be in ]0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('clustering', 'cc_merge') >=
0) and (self.parser.getfloat('clustering', 'cc_merge') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["cc_merge in [validating] should be in [0,1]"],
'error', logger)
sys.exit(0)
# test = (self.parser.getfloat('clustering', 'ignored_mixtures') >= 0) and (self.parser.getfloat('clustering', 'ignored_mixtures') <= 1)
# if not test:
# if comm.rank == 0:
# print_and_log(["ignored_mixtures in [validating] should be in [0,1]"], 'error', logger)
# sys.exit(0)
test = (self.parser.getfloat('data', 'memory_usage') >
0) and (self.parser.getfloat('data', 'memory_usage') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["memory_usage in [data] should be in ]0,1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'auto_mode') >=
0) and (self.parser.getfloat('merging', 'auto_mode') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["auto_mode in [merging] should be in [0, 1]"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'noise_limit') >= 0)
if not test:
if comm.rank == 0:
print_and_log(["noise_limit in [merging] should be > 0"],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('merging', 'sparsity_limit') <= 1)
if not test:
if comm.rank == 0:
print_and_log(["sparsity_limit in [merging] should be < 1"],
'error', logger)
sys.exit(0)
test = (self.parser.getint('detection', 'oversampling_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["oversampling_factor in [detection] should be positive["],
'error', logger)
sys.exit(0)
test = (self.parser.getfloat('detection', 'smoothing_factor') >= 0)
if not test:
if comm.rank == 0:
print_and_log(
["smoothing_factor in [detection] should be positive["],
'error', logger)
sys.exit(0)
test = (not self.parser.getboolean('data', 'overwrite')
and not self.parser.getboolean('filtering', 'filter'))
if test:
if comm.rank == 0:
print_and_log(
["If no filtering, then overwrite should be True"],
'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering', 'validating', 'triggers']:
test = self.parser.get('clustering',
'make_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"make_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
fileformats = ['png', 'pdf', 'eps', 'jpg', '', 'None']
for section in ['clustering']:
test = self.parser.get('clustering',
'debug_plots').lower() in fileformats
if not test:
if comm.rank == 0:
print_and_log([
"debug_plots in [%s] should be in %s" %
(section, str(fileformats))
], 'error', logger)
sys.exit(0)
methods = [
'distance', 'dip', 'folding', 'nd-folding', 'bhatta', 'nd-bhatta'
]
test = self.parser.get('clustering',
'merging_method').lower() in methods
if not test:
if comm.rank == 0:
print_and_log([
"merging_method in [%s] should be in %s" %
(section, str(methods))
], 'error', logger)
sys.exit(0)
if self.parser.get('clustering', 'merging_param').lower() == 'default':
method = self.parser.get('clustering', 'merging_method').lower()
if method == 'dip':
self.parser.set('clustering', 'merging_param', '0.5')
elif method == 'distance':
self.parser.set('clustering', 'merging_param', '3')
elif method in ['folding', 'nd-folding']:
self.parser.set('clustering', 'merging_param', '1e-9')
elif method in ['bhatta', 'nd-bhatta']:
self.parser.set('clustering', 'merging_param', '2')
has_same_elec = self.parser.has_option('clustering', 'sim_same_elec')
has_dip_thresh = self.parser.has_option('clustering', 'dip_threshold')
if has_dip_thresh:
dip_threshold = self.parser.getfloat('clustering', 'dip_threshold')
if has_dip_thresh and dip_threshold > 0:
if comm.rank == 0:
print_and_log([
"dip_threshold in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"By default a nd-bhatta merging method with param 3 is assumed"
], 'info', logger)
self.parser.set('clustering', 'merging_param', str(3))
self.parser.set('clustering', 'merging_method', 'distance')
elif has_same_elec:
sim_same_elec = self.parser.get('clustering', 'sim_same_elec')
if comm.rank == 0:
print_and_log([
"sim_same_elec in [clustering] is deprecated since 0.8.4",
"and you should now use merging_method and merging_param",
"Please upgrade your parameter file to a more recent version",
"Meanwhile a distance merging method with param %s is assumed"
% sim_same_elec
], 'info', logger)
self.parser.set('clustering', 'merging_param', sim_same_elec)
self.parser.set('clustering', 'merging_method', 'distance')
dispersion = self.parser.get('clustering', 'dispersion').replace(
'(', '').replace(')', '').split(',')
dispersion = map(float, dispersion)
test = (0 < dispersion[0]) and (0 < dispersion[1])
if not test:
if comm.rank == 0:
print_and_log([
"min and max dispersions in [clustering] should be positive"
], 'error', logger)
sys.exit(0)
pcs_export = ['prompt', 'none', 'all', 'some']
test = self.parser.get('converting',
'export_pcs').lower() in pcs_export
if not test:
if comm.rank == 0:
print_and_log([
"export_pcs in [converting] should be in %s" %
str(pcs_export)
], 'error', logger)
sys.exit(0)
else:
if self.parser.get('converting', 'export_pcs').lower() == 'none':
self.parser.set('converting', 'export_pcs', 'n')
elif self.parser.get('converting', 'export_pcs').lower() == 'some':
self.parser.set('converting', 'export_pcs', 's')
elif self.parser.get('converting', 'export_pcs').lower() == 'all':
self.parser.set('converting', 'export_pcs', 'a')
if self.parser.getboolean('detection', 'hanning'):
if comm.rank == 0:
print_and_log(["Hanning filtering is activated"], 'debug',
logger)