Example #1
0
def distribute_observation(detectors,
                           observations,
                           rank=None,
                           comm=MPI.COMM_WORLD):
    size = comm.size
    if size == 1:
        return detectors.copy(), list(observations)

    if rank is None:
        rank = comm.Get_rank()
    nthreads = omp_num_threads()
    ndetectors = np.sum(~detectors)
    nobservations = len(observations)

    # number of observations. They should approximatively be of the same length
    nx = nobservations

    # number of detectors, grouped by the number of cpu cores
    ny = int(np.ceil(ndetectors / nthreads))

    # we start with the minimum blocksize and increase it until we find a
    # configuration that covers all the observations
    blocksize = int(np.ceil(nx * ny / size))
    while True:
        # by looping over x first, we favor larger numbers of detectors and
        # fewer numbers of observations per processor, to minimise inter-
        # processor communication in case of correlations between
        # detectors
        for xblocksize in range(1, blocksize + 1):
            if blocksize / xblocksize != blocksize // xblocksize:
                continue
            yblocksize = int(blocksize // xblocksize)
            nx_block = int(np.ceil(nx / xblocksize))
            ny_block = int(np.ceil(ny / yblocksize))
            if nx_block * ny_block <= size:
                break
        if nx_block * ny_block <= size:
            break
        blocksize += 1

    ix = rank // ny_block
    iy = rank % ny_block

    # check that the processor has something to do
    if ix >= nx_block:
        idetector = slice(0, 0)
        iobservation = slice(0, 0)
    else:
        idetector = slice(iy * yblocksize * nthreads,
                          (iy + 1) * yblocksize * nthreads)
        iobservation = slice(ix * xblocksize, (ix + 1) * xblocksize)

    detectors_ = detectors.copy()
    igood = np.where(~detectors_.ravel())[0]
    detectors_.ravel()[igood[0:idetector.start]] = True
    detectors_.ravel()[igood[idetector.stop:]] = True
    observations_ = observations[iobservation]

    return detectors_, observations_
Example #2
0
def distribute_observation(detectors, observations, rank=None,
                           comm=MPI.COMM_WORLD):
    size = comm.size
    if size == 1:
        return detectors.copy(), list(observations)

    if rank is None:
        rank = comm.Get_rank()
    nthreads = omp_num_threads()
    ndetectors = np.sum(~detectors)
    nobservations = len(observations)

    # number of observations. They should approximatively be of the same length
    nx = nobservations

    # number of detectors, grouped by the number of cpu cores
    ny = int(np.ceil(ndetectors / nthreads))

    # we start with the minimum blocksize and increase it until we find a
    # configuration that covers all the observations
    blocksize = int(np.ceil(nx * ny / size))
    while True:
        # by looping over x first, we favor larger numbers of detectors and
        # fewer numbers of observations per processor, to minimise inter-
        # processor communication in case of correlations between
        # detectors
        for xblocksize in range(1, blocksize+1):
            if blocksize / xblocksize != blocksize // xblocksize:
                continue
            yblocksize = int(blocksize // xblocksize)
            nx_block = int(np.ceil(nx / xblocksize))
            ny_block = int(np.ceil(ny / yblocksize))
            if nx_block * ny_block <= size:
                break
        if nx_block * ny_block <= size:
            break
        blocksize += 1

    ix = rank // ny_block
    iy = rank % ny_block

    # check that the processor has something to do
    if ix >= nx_block:
        idetector = slice(0, 0)
        iobservation = slice(0, 0)
    else:
        idetector = slice(iy * yblocksize * nthreads, (iy+1) * yblocksize *
                          nthreads)
        iobservation = slice(ix * xblocksize, (ix+1) * xblocksize)

    detectors_ = detectors.copy()
    igood = np.where(~detectors_.ravel())[0]
    detectors_.ravel()[igood[0:idetector.start]] = True
    detectors_.ravel()[igood[idetector.stop:]] = True
    observations_ = observations[iobservation]

    return detectors_, observations_
Example #3
0
 def func(env):
     global counter
     with env:
         nthreads = os.getenv("OMP_NUM_THREADS")
         expected = omp_num_threads()
         with pool_threading() as pool:
             assert_equal(int(os.environ["OMP_NUM_THREADS"]), 1)
             if mkl is not None:
                 assert_equal(mkl.get_max_threads(), 1)
             counter = 0
             pool.map(func_thread, range(pool._processes))
         assert_equal(os.getenv("OMP_NUM_THREADS"), nthreads)
         if mkl is not None:
             assert_equal(mkl.get_max_threads(), mkl_nthreads)
         assert_equal(counter, expected)
     assert_not_in("OMP_NUM_THREADS", os.environ)
Example #4
0
 def func(env):
     global counter
     with env:
         nthreads = os.getenv('OMP_NUM_THREADS')
         expected = omp_num_threads()
         with pool_threading() as pool:
             assert_equal(int(os.environ['OMP_NUM_THREADS']), 1)
             if mkl is not None:
                 assert_equal(mkl.get_max_threads(), 1)
             counter = 0
             pool.map(func_thread, range(pool._processes))
         assert_equal(os.getenv('OMP_NUM_THREADS'), nthreads)
         if mkl is not None:
             assert_equal(mkl.get_max_threads(), mkl_nthreads)
         assert_equal(counter, expected)
     assert_not_in('OMP_NUM_THREADS', os.environ)