def test_patch_for_similarities(params, extension): file_out_suff = params.get('data', 'file_out_suff') template_file = file_out_suff + '.templates%s.hdf5' % extension if os.path.exists(template_file): version = io.load_data(params, 'version', extension) else: print_and_log(['No templates found! Check suffix?'], 'error', logger) sys.exit(0) if version is not None: if (StrictVersion(version) >= StrictVersion('0.6.0')): return True else: print_and_log(["Version is below 0.6.0"], 'debug', logger) return False
def apply_patch_for_similarities(params, extension): if not test_patch_for_similarities(params, extension): file_out_suff = params.get('data', 'file_out_suff') hdf5_compress = params.getboolean('data', 'hdf5_compress') blosc_compress = params.getboolean('data', 'blosc_compress') N_tm = io.load_data(params, 'nb_templates', extension) N_half = int(N_tm // 2) N_t = params.getint('detection', 'N_t') duration = 2 * N_t - 1 if comm.rank == 0: print_and_log(["Fixing overlaps from 0.5.XX..."], 'default', logger) maxlag = numpy.zeros((N_half, N_half), dtype=numpy.int32) maxoverlap = numpy.zeros((N_half, N_half), dtype=numpy.float32) to_explore = numpy.arange(N_half - 1)[comm.rank::comm.size] if comm.rank == 0: to_explore = get_tqdm_progressbar(to_explore) if not SHARED_MEMORY: over_x, over_y, over_data, over_shape = io.load_data( params, 'overlaps-raw', extension=extension) else: over_x, over_y, over_data, over_shape = io.load_data_memshared( params, 'overlaps-raw', extension=extension) for i in to_explore: idx = numpy.where((over_x >= i * N_tm + i + 1) & (over_x < (i * N_tm + N_half)))[0] local_x = over_x[idx] - (i * N_tm + i + 1) data = numpy.zeros((N_half - (i + 1), duration), dtype=numpy.float32) data[local_x, over_y[idx]] = over_data[idx] maxlag[i, i + 1:] = N_t - numpy.argmax(data, 1) maxlag[i + 1:, i] = -maxlag[i, i + 1:] maxoverlap[i, i + 1:] = numpy.max(data, 1) maxoverlap[i + 1:, i] = maxoverlap[i, i + 1:] #Now we need to sync everything across nodes maxlag = gather_array(maxlag, comm, 0, 1, 'int32', compress=blosc_compress) if comm.rank == 0: maxlag = maxlag.reshape(comm.size, N_half, N_half) maxlag = numpy.sum(maxlag, 0) maxoverlap = gather_array(maxoverlap, comm, 0, 1, 'float32', compress=blosc_compress) if comm.rank == 0: maxoverlap = maxoverlap.reshape(comm.size, N_half, N_half) maxoverlap = numpy.sum(maxoverlap, 0) if comm.rank == 0: myfile2 = h5py.File(file_out_suff + '.templates%s.hdf5' % extension, 'r+', libver='earliest') for key in ['maxoverlap', 'maxlag', 'version']: if key in myfile2.keys(): myfile2.pop(key) myfile2.create_dataset('version', data=numpy.array( circus.__version__.split('.'), dtype=numpy.int32)) if hdf5_compress: myfile2.create_dataset('maxlag', data=maxlag, compression='gzip') myfile2.create_dataset('maxoverlap', data=maxoverlap, compression='gzip') else: myfile2.create_dataset('maxlag', data=maxlag) myfile2.create_dataset('maxoverlap', data=maxoverlap) myfile2.close()