def test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
template_file = file_out_suff + '.templates%s.hdf5' % extension
if os.path.exists(template_file):
version = io.load_data(params, 'version', extension)
else:
print_and_log(['No templates found! Check suffix?'], 'error', logger)
sys.exit(0)
if version is not None:
if (StrictVersion(version) >= StrictVersion('0.6.0')):
return True
else:
print_and_log(["Version is below 0.6.0"], 'debug', logger)
return False
def apply_patch_for_similarities(params, extension):
if not test_patch_for_similarities(params, extension):
file_out_suff = params.get('data', 'file_out_suff')
hdf5_compress = params.getboolean('data', 'hdf5_compress')
blosc_compress = params.getboolean('data', 'blosc_compress')
N_tm = io.load_data(params, 'nb_templates', extension)
N_half = int(N_tm // 2)
N_t = params.getint('detection', 'N_t')
duration = 2 * N_t - 1
if comm.rank == 0:
print_and_log(["Fixing overlaps from 0.5.XX..."], 'default',
logger)
maxlag = numpy.zeros((N_half, N_half), dtype=numpy.int32)
maxoverlap = numpy.zeros((N_half, N_half), dtype=numpy.float32)
to_explore = numpy.arange(N_half - 1)[comm.rank::comm.size]
if comm.rank == 0:
to_explore = get_tqdm_progressbar(to_explore)
if not SHARED_MEMORY:
over_x, over_y, over_data, over_shape = io.load_data(
params, 'overlaps-raw', extension=extension)
else:
over_x, over_y, over_data, over_shape = io.load_data_memshared(
params, 'overlaps-raw', extension=extension)
for i in to_explore:
idx = numpy.where((over_x >= i * N_tm + i + 1)
& (over_x < (i * N_tm + N_half)))[0]
local_x = over_x[idx] - (i * N_tm + i + 1)
data = numpy.zeros((N_half - (i + 1), duration),
dtype=numpy.float32)
data[local_x, over_y[idx]] = over_data[idx]
maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1)
maxlag[i + 1:, i] = -maxlag[i, i + 1:]
maxoverlap[i, i + 1:] = numpy.max(data, 1)
maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:]
#Now we need to sync everything across nodes
maxlag = gather_array(maxlag,
comm,
0,
1,
'int32',
compress=blosc_compress)
if comm.rank == 0:
maxlag = maxlag.reshape(comm.size, N_half, N_half)
maxlag = numpy.sum(maxlag, 0)
maxoverlap = gather_array(maxoverlap,
comm,
0,
1,
'float32',
compress=blosc_compress)
if comm.rank == 0:
maxoverlap = maxoverlap.reshape(comm.size, N_half, N_half)
maxoverlap = numpy.sum(maxoverlap, 0)
if comm.rank == 0:
myfile2 = h5py.File(file_out_suff +
'.templates%s.hdf5' % extension,
'r+',
libver='earliest')
for key in ['maxoverlap', 'maxlag', 'version']:
if key in myfile2.keys():
myfile2.pop(key)
myfile2.create_dataset('version',
data=numpy.array(
circus.__version__.split('.'),
dtype=numpy.int32))
if hdf5_compress:
myfile2.create_dataset('maxlag',
data=maxlag,
compression='gzip')
myfile2.create_dataset('maxoverlap',
data=maxoverlap,
compression='gzip')
else:
myfile2.create_dataset('maxlag', data=maxlag)
myfile2.create_dataset('maxoverlap', data=maxoverlap)
myfile2.close()
