def testHandleValue(self):
typemap = {ctypes.sizeof(ctypes.c_uint32): ctypes.c_uint32,
ctypes.sizeof(ctypes.c_uint64): ctypes.c_uint64}
for obj in self.objects:
uintptr_t = typemap[MPI._sizeof(obj)]
handle = uintptr_t.from_address(MPI._addressof(obj))
self.assertEqual(handle.value, MPI._handleof(obj))
def init(comm=None):
"""A function that initializes Horovod.
Args:
comm: List specifying ranks for the communicator, relative to the MPI_COMM_WORLD
communicator OR the MPI communicator to use. Given communicator will be duplicated.
If None, Horovod will use MPI_COMM_WORLD Communicator.
"""
if comm is None:
comm = []
atexit.register(shutdown)
if not isinstance(comm, list):
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
MPI_COMMON_LIB_CTYPES.horovod_init_comm.argtypes = [MPI_Comm]
comm_obj = MPI_Comm.from_address(MPI._addressof(comm))
return MPI_COMMON_LIB_CTYPES.horovod_init_comm(comm_obj)
else:
comm_size = len(comm)
return MPI_COMMON_LIB_CTYPES.horovod_init(
(ctypes.c_int * comm_size)(*comm), ctypes.c_int(comm_size))
def init(self, comm=None):
"""A function that initializes Horovod.
Args:
comm: List specifying ranks for the communicator, relative to the MPI_COMM_WORLD
communicator OR the MPI communicator to use. Given communicator will be duplicated.
If None, Horovod will use MPI_COMM_WORLD Communicator.
"""
if comm is None:
comm = []
atexit.register(self.shutdown)
if not isinstance(comm, list):
mpi_built = self.MPI_LIB_CTYPES.horovod_mpi_built()
if not bool(mpi_built):
raise ValueError(
"Horovod has not been built with MPI support. Ensure MPI is installed and "
"reinstall Horovod with HOROVOD_WITH_MPI=1 to debug the build error.")
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
self.MPI_LIB_CTYPES.horovod_init_comm.argtypes = [MPI_Comm]
comm_obj = MPI_Comm.from_address(MPI._addressof(comm))
self.MPI_LIB_CTYPES.horovod_init_comm(comm_obj)
else:
comm_size = len(comm)
self.MPI_LIB_CTYPES.horovod_init(
(ctypes.c_int * comm_size)(*comm), ctypes.c_int(comm_size))
def _comm_process_set_id(self, comm: MPI.Comm) -> int:
""" Returns the (previously registered) process set id corresponding to the MPI communicator comm. """
if not self.mpi_built():
raise ValueError(
"Horovod has not been built with MPI support. Ensure MPI is installed and "
"reinstall Horovod with HOROVOD_WITH_MPI=1 to debug the build error."
)
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
self.MPI_LIB_CTYPES.horovod_comm_process_set.argtypes = [MPI_Comm]
comm_obj = MPI_Comm.from_address(MPI._addressof(comm))
result = int(self.MPI_LIB_CTYPES.horovod_comm_process_set(comm_obj))
if result == self.HOROVOD_PROCESS_SET_ERROR_INIT:
raise ValueError(
'Horovod has not been initialized or MPI has not been enabled; use hvd.init().'
)
elif result == self.HOROVOD_PROCESS_SET_ERROR_UNKNOWN_SET:
raise ValueError(
'MPI communicator does not correspond to any registered process set.'
)
return result
def testHandleValue(self):
typemap = {ctypes.sizeof(ctypes.c_uint32): ctypes.c_uint32,
ctypes.sizeof(ctypes.c_uint64): ctypes.c_uint64}
for obj in self.objects:
uintptr_t = typemap[MPI._sizeof(obj)]
handle = uintptr_t.from_address(MPI._addressof(obj))
self.assertEqual(handle.value, MPI._handleof(obj))
class MPICommObject(Object):
name = 'comm'
# See https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
if MPI._sizeof(MPI.Comm) == sizeof(c_int):
dtype = type('MPI_Comm', (c_int, ), {})
else:
dtype = type('MPI_Comm', (c_void_p, ), {})
def __init__(self, comm=None):
if comm is None:
# Should only end up here upon unpickling
comm = MPI.COMM_WORLD
comm_ptr = MPI._addressof(comm)
comm_val = self.dtype.from_address(comm_ptr)
self.value = comm_val
self.comm = comm
def _arg_values(self, *args, **kwargs):
grid = kwargs.get('grid', None)
# Update `comm` based on object attached to `grid`
if grid is not None:
return grid.distributor._obj_comm._arg_defaults()
else:
return self._arg_defaults()
# Pickling support
_pickle_args = []
def init(self, comm=None):
"""A function that initializes Horovod.
Args:
comm: List specifying ranks for the communicator, relative to the MPI_COMM_WORLD
communicator OR the MPI communicator to use. Given communicator will be duplicated.
If None, Horovod will use MPI_COMM_WORLD Communicator.
"""
if comm is None:
comm = []
atexit.register(self.shutdown)
if not isinstance(comm, list):
mpi_enabled = self.MPI_LIB_CTYPES.horovod_mpi_enabled()
if not bool(mpi_enabled):
raise ValueError(
'Horovod MPI is not enabled; Please make sure it\'s installed and enabled.'
)
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
self.MPI_LIB_CTYPES.horovod_init_comm.argtypes = [MPI_Comm]
comm_obj = MPI_Comm.from_address(MPI._addressof(comm))
self.MPI_LIB_CTYPES.horovod_init_comm(comm_obj)
else:
comm_size = len(comm)
self.MPI_LIB_CTYPES.horovod_init((ctypes.c_int * comm_size)(*comm),
ctypes.c_int(comm_size))
def init(self, calling_realm):
self.realm = calling_realm
# Build a communicator mpi4py python object from the
# handle returned by the CPL_init function.
try:
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
#Some versions of MPI4py have no _sizeof method.
except AttributeError:
MPI_Comm = c_int
# Call create comm
returned_realm_comm = c_int()
self._py_init(calling_realm, byref(returned_realm_comm))
# Use an intracomm object as the template and override value
newcomm = MPI.Intracomm()
newcomm_ptr = MPI._addressof(newcomm)
comm_val = MPI_Comm.from_address(newcomm_ptr)
comm_val.value = returned_realm_comm.value
self.COMM = newcomm
return newcomm
def init(self, comm=None):
"""A function that initializes Horovod.
Args:
comm: List specifying ranks for the communicator, relative to the
MPI_COMM_WORLD communicator OR the MPI communicator to use. Given
communicator will be duplicated. If None, Horovod will use
MPI_COMM_WORLD Communicator.
"""
if comm is None:
comm = []
atexit.register(self.shutdown)
if not isinstance(comm, list):
from mpi4py import MPI # pylint: disable=import-outside-toplevel
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int): # pylint: disable=protected-access
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
self.MPI_LIB_CTYPES.horovod_init_comm.argtypes = [MPI_Comm]
comm_obj = MPI_Comm.from_address(MPI._addressof(comm)) # pylint: disable=protected-access
self.MPI_LIB_CTYPES.horovod_init_comm(comm_obj)
else:
comm_size = len(comm)
self.MPI_LIB_CTYPES.horovod_init((ctypes.c_int * comm_size)(*comm),
ctypes.c_int(comm_size))
def testHandleValue(self):
ffi = cffi.FFI()
typemap = {ffi.sizeof('uint32_t'): 'uint32_t',
ffi.sizeof('uint64_t'): 'uint64_t',}
for obj in self.objects:
uintptr_t = typemap[MPI._sizeof(obj)]
handle = ffi.cast(uintptr_t+'*', MPI._addressof(obj))[0]
self.assertEqual(handle, MPI._handleof(obj))
def __init__(self, name="", cmdargs=None, comm=None):
# load liblammps.so by default
# if name = "g++", load liblammps_g++.so
try:
if not name: self.lib = CDLL("liblammps.so", RTLD_GLOBAL)
else: self.lib = CDLL("liblammps_%s.so" % name, RTLD_GLOBAL)
except:
type, value, tb = sys.exc_info()
traceback.print_exception(type, value, tb)
raise OSError, "Could not load LAMMPS dynamic library"
# create an instance of LAMMPS
# don't know how to pass an MPI communicator from PyPar
# no_mpi call lets LAMMPS use MPI_COMM_WORLD
# cargs = array of C strings from args
if cmdargs:
cmdargs.insert(0, "lammps.py")
narg = len(cmdargs)
cargs = (c_char_p * narg)(*cmdargs)
self.lmp = c_void_p()
if comm is None:
self.lib.lammps_open_no_mpi(narg, cargs, byref(self.lmp))
else:
if MPI._sizeof(MPI.Comm) == sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
comm_ptr = MPI._addressof(comm)
comm_val = MPI_Comm.from_address(comm_ptr)
self.lib.lammps_open(narg, cargs, comm_val, byref(self.lmp))
else:
self.lmp = c_void_p()
if comm is None:
self.lib.lammps_open_no_mpi(0, None, byref(self.lmp))
else:
if MPI._sizeof(MPI.Comm) == sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
comm_ptr = MPI._addressof(comm)
comm_val = MPI_Comm.from_address(comm_ptr)
self.lib.lammps_open(0, None, comm_val, byref(self.lmp))
def testHandleAdress(self):
typemap = {ctypes.sizeof(ctypes.c_int): ctypes.c_int,
ctypes.sizeof(ctypes.c_void_p): ctypes.c_void_p}
for obj in self.objects:
handle_t = typemap[MPI._sizeof(obj)]
oldobj = obj
newobj = type(obj)()
handle_old = handle_t.from_address(MPI._addressof(oldobj))
handle_new = handle_t.from_address(MPI._addressof(newobj))
handle_new.value = handle_old.value
self.assertEqual(obj, newobj)
def testHandleAdress(self):
typemap = {ctypes.sizeof(ctypes.c_int): ctypes.c_int,
ctypes.sizeof(ctypes.c_void_p): ctypes.c_void_p}
for obj in self.objects:
handle_t = typemap[MPI._sizeof(obj)]
oldobj = obj
newobj = type(obj)()
handle_old = handle_t.from_address(MPI._addressof(oldobj))
handle_new = handle_t.from_address(MPI._addressof(newobj))
handle_new.value = handle_old.value
self.assertEqual(obj, newobj)
def get_task_comm():
from mpi4py import MPI
import ctypes
# print("turbine_helpers.task_comm: %i" % task_comm)
# sys.stdout.flush()
mpi4py_comm = MPI.Intracomm()
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
# MPICH
comm_int = ctypes.c_int
mpi4py_comm_ptr = comm_int.from_address(MPI._addressof(mpi4py_comm))
mpi4py_comm_ptr.value = task_comm
elif MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_void_p):
# OpenMPI
comm_pointer = ctypes.c_void_p
mpi4py_comm = MPI.Intracomm()
handle = comm_pointer.from_address(MPI._addressof(mpi4py_comm))
handle.value = task_comm
return mpi4py_comm
def comm_ptr(comm):
"""Get the pointer to the communicator
"""
# see https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
comm = get_comm(comm)
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == c.sizeof(c.c_int):
MPI_Comm = c.c_int
else:
# must be pointer
MPI_Comm = c.c_void_p
return c.byref(MPI_Comm.from_address(MPI._addressof(comm)))
def _C_comm(self):
"""
A :class:`Object` wrapping an MPI communicator.
Extracted from: ::
https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
"""
from devito.types import CompositeObject
ptype = c_int if MPI._sizeof(self._comm) == sizeof(c_int) else c_void_p
obj = CompositeObject('comm', 'MPI_Comm', ptype, [])
comm_ptr = MPI._addressof(self._comm)
comm_val = obj.dtype.from_address(comm_ptr)
obj.value = comm_val
return obj
def _C_comm(self):
"""
A :class:`Object` wrapping an MPI communicator.
Extracted from: ::
https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
"""
from devito.types import Object
if MPI._sizeof(self._comm) == sizeof(c_int):
ctype = type('MPI_Comm', (c_int, ), {})
else:
ctype = type('MPI_Comm', (c_void_p, ), {})
comm_ptr = MPI._addressof(self._comm)
comm_val = ctype.from_address(comm_ptr)
return Object(name='comm', dtype=ctype, value=comm_val)
def testHandleAddress(self):
ffi = cffi.FFI()
typemap = {ffi.sizeof('int'): 'int', ffi.sizeof('void*'): 'void*'}
typename = lambda t: t.__name__.rsplit('.', 1)[-1]
for tp in self.mpitypes:
handle_t = typemap[MPI._sizeof(tp)]
mpi_t = 'MPI_' + typename(tp)
ffi.cdef("typedef %s %s;" % (handle_t, mpi_t))
for obj in self.objects:
if isinstance(obj, MPI.Comm):
mpi_t = 'MPI_Comm'
else:
mpi_t = 'MPI_' + typename(type(obj))
oldobj = obj
newobj = type(obj)()
handle_old = ffi.cast(mpi_t + '*', MPI._addressof(oldobj))
handle_new = ffi.cast(mpi_t + '*', MPI._addressof(newobj))
handle_new[0] = handle_old[0]
self.assertEqual(oldobj, newobj)
class MPICommObject(Object):
name = 'comm'
# See https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
if MPI._sizeof(MPI.Comm) == sizeof(c_int):
dtype = type('MPI_Comm', (c_int, ), {})
else:
dtype = type('MPI_Comm', (c_void_p, ), {})
def __init__(self, comm=None):
if comm is None:
# Should only end up here upon unpickling
comm = MPI.COMM_WORLD
comm_ptr = MPI._addressof(comm)
comm_val = self.dtype.from_address(comm_ptr)
self.value = comm_val
# Pickling support
_pickle_args = []
def __init__(self, comm, dest, outdir):
# For python 2 compatibility
super(DataSpacesPBTClient, self).__init__(comm, dest, outdir)
path = os.path.dirname(os.path.abspath(__file__))
self.lib = ctypes.cdll.LoadLibrary("{}/libpbt_ds.so".format(path))
# different mpi implementation use different types for
# MPI_Comm, this determines which type to use
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
self.mpi_comm_type = ctypes.c_int
else:
self.mpi_comm_type = ctypes.c_void_p
group = comm.Get_group()
newgroup = group.Excl([dest])
ds_comm = comm.Create(newgroup)
self.mpi_comm_self = self.make_comm_arg(MPI.COMM_SELF)
mpi_comm_ds = self.make_comm_arg(ds_comm)
world_size = ds_comm.Get_size()
self.lib.pbt_ds_init(ctypes.c_int(world_size), mpi_comm_ds)
def init(self, calling_realm):
# Build a communicator mpi4py python object from the
# handle returned by the CPL_init function.
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
# Call create comm
returned_realm_comm = c_int()
self._py_init(calling_realm, byref(returned_realm_comm))
# Use an intracomm object as the template and override value
newcomm = MPI.Intracomm()
newcomm_ptr = MPI._addressof(newcomm)
comm_val = MPI_Comm.from_address(newcomm_ptr)
comm_val.value = returned_realm_comm.value
return newcomm
def testHandleAddress(self):
ffi = cffi.FFI()
typemap = {ffi.sizeof('int'): 'int',
ffi.sizeof('void*'): 'void*'}
typename = lambda t: t.__name__.rsplit('.', 1)[-1]
for tp in self.mpitypes:
handle_t = typemap[MPI._sizeof(tp)]
mpi_t = 'MPI_' + typename(tp)
ffi.cdef("typedef %s %s;" % (handle_t, mpi_t))
for obj in self.objects:
if isinstance(obj, MPI.Comm):
mpi_t = 'MPI_Comm'
else:
mpi_t = 'MPI_' + typename(type(obj))
oldobj = obj
newobj = type(obj)()
handle_old = ffi.cast(mpi_t+'*', MPI._addressof(oldobj))
handle_new = ffi.cast(mpi_t+'*', MPI._addressof(newobj))
handle_new[0] = handle_old[0]
self.assertEqual(oldobj, newobj)
def diffusion_2d_mpi(v, iterations, kappa):
#libdiffuse = load_lib()
libdiffuse = load_lib_alt()
# type stuff
from ctypes import c_double, c_int
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
print "MPI_COMM is an int"
else:
MPI_Comm = ctypes.c_void_p
print "MPI_COMM is a void pointer"
float64_array_2d = np.ctypeslib.ndpointer(dtype=c_double, ndim=2,
flags="contiguous")
# MPI_Comm
comm = MPI.COMM_WORLD
comm_ptr = MPI._addressof(comm)
comm_val = MPI_Comm.from_address(comm_ptr)
height, width = v.shape
libdiffuse.solve_2d_mpi.restype = None
libdiffuse.solve_2d_mpi.argtypes = [float64_array_2d,
c_int,
c_int,
c_double,
c_int,
MPI_Comm]
libdiffuse.solve_2d_mpi(v,
c_int(height),
c_int(width),
c_double(kappa),
c_int(iterations),
comm_val
)
"""
Python wrapper for LJMD C code
Author: Krister Jazz Urog
Date: 17 February, 2020
"""
from ctypes import *
import sys
import os
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
class MDSYS_T(Structure):
"""
Molecular dynamics system structure object wrapper for a similar C struct.
-----------------------------------------------
Parameters:
natoms = number of atoms
nfi = current iteration step
nsteps = total number of iteration steps
dt = time step
mass = mass of atom
epsilon = constant
sigma = constant
box = box size
import numba
import ctypes
import numpy as np
import platform
from mpi4py import MPI
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
_MPI_Comm_t = ctypes.c_int
else:
_MPI_Comm_t = ctypes.c_void_p
if MPI._sizeof(MPI.Datatype) == ctypes.sizeof(ctypes.c_int):
_MPI_Datatype_t = ctypes.c_int
else:
_MPI_Datatype_t = ctypes.c_void_p
_MPI_Status_ptr_t = ctypes.c_void_p
if platform.system() == 'Linux':
lib = 'libmpi.so'
elif platform.system() == 'Windows':
lib = 'msmpi.dll'
elif platform.system() == 'Darwin':
lib = 'libmpi.dylib'
else:
raise NotImplementedError()
libmpi = ctypes.CDLL(lib)
_MPI_Initialized = libmpi.MPI_Initialized
_MPI_Initialized.restype = ctypes.c_int
_MPI_Initialized.argtypes = [ctypes.c_void_p]
class CPL:
# Shared attribute containing the library
CFD_REALM = 1
MD_REALM = 2
GATHER_SCATTER = 1
SEND_RECEIVE = 2
NULL_REALM = 0
_libname = "libcpl"
try:
_lib_path = os.environ["CPL_LIBRARY_PATH"]
if os.path.exists(_lib_path + "/" + _libname + ".so"):
_cpl_lib = load_library(_libname, _lib_path)
else:
raise CPLLibraryNotFound(
"Compiled CPL library libcpl.so not found at " + _lib_path +
"/" + _libname + ".so")
except KeyError as e:
print(
"CPL info: ",
"CPL_LIBRARY_PATH not defined. Looking in system directories...")
try:
_cpl_lib = cdll.LoadLibrary(_libname + ".so")
print("CPL info: ", "Success!")
except OSError as e:
raise CPLLibraryNotFound("Library libcpl.so not found!")
#TODO: Check this
#time.sleep(2)
#MPI.COMM_WORLD.Abort(errorcode=1)
# Check for JSON support by cheking if load_param_file symbol exists
JSON_SUPPORT = True
try:
_cpl_lib.CPLC_load_param_file
except:
JSON_SUPPORT = False
def __init__(self):
self._var = POINTER(POINTER(c_char_p))
self.realm = None
# py_test_python function
py_test_python = _cpl_lib.CPLC_test_python
py_test_python.argtypes = \
[c_int,
c_double,
c_bool,
ndpointer(np.int32, ndim=2, flags='aligned, f_contiguous'),
ndpointer(np.float64, ndim=2, flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(2,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(2,), flags='aligned, f_contiguous')]
@abortMPI
def test_python(self, int_p, doub_p, bool_p, int_pptr, doub_pptr):
int_pptr_dims = np.array(int_pptr.shape, order='F', dtype=np.int32)
doub_pptr_dims = np.array(doub_pptr.shape, order='F', dtype=np.int32)
self.py_test_python(int_p, doub_p, bool_p, int_pptr, doub_pptr,
int_pptr_dims, doub_pptr_dims)
_py_init = _cpl_lib.CPLC_init
#OpenMPI comm greater than c_int
try:
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int):
_py_init.argtypes = [c_int, POINTER(c_int)]
else:
excptstr = "Problem is in create_comm wrapper, as the OpenMPI COMM handle is not "
excptstr += "an integer, c_void_p should be used so C bindings needs something like **void"
excptstr += "(No idea what to do in the Fortran code, maybe MPI_COMM_f2C required)"
raise OpenMPI_Not_Supported(excptstr)
_py_init.argtypes = [c_int, POINTER(c_void_p)]
except AttributeError:
_py_init.argtypes = [c_int, POINTER(c_int)]
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def init(self, calling_realm):
self.realm = calling_realm
# Build a communicator mpi4py python object from the
# handle returned by the CPL_init function.
try:
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int):
MPI_Comm = c_int
else:
MPI_Comm = c_void_p
#Some versions of MPI4py have no _sizeof method.
except AttributeError:
MPI_Comm = c_int
# Call create comm
returned_realm_comm = c_int()
self._py_init(calling_realm, byref(returned_realm_comm))
# Use an intracomm object as the template and override value
newcomm = MPI.Intracomm()
newcomm_ptr = MPI._addressof(newcomm)
comm_val = MPI_Comm.from_address(newcomm_ptr)
comm_val.value = returned_realm_comm.value
self.COMM = newcomm
return newcomm
if JSON_SUPPORT:
_py_load_param_file = _cpl_lib.CPLC_load_param_file
_py_load_param_file.argtypes = [c_char_p]
@abortMPI
def load_param_file(self, fname):
self._py_load_param_file(c_char_p(fname), c_int(len(fname)))
if JSON_SUPPORT:
_py_close_param_file = _cpl_lib.CPLC_close_param_file
@abortMPI
def close_param_file(self):
self._py_close_param_file()
@abortMPI
def get_file_var(self, section, var_name, var_type):
try:
fun_name = _CPL_GET_FILE_VARS[var_type][0]
var_ctype = _CPL_GET_FILE_VARS[var_type][1]
fun = getattr(self._cpl_lib, "CPLC_" + fun_name)
fun.argtypes = [c_char_p, c_char_p, POINTER(var_ctype)]
except KeyError:
print("CPL-ERROR: CPL Library function '" + str(fun_name) +
"' not found!")
raise KeyError
else:
self._var = var_ctype()
if ("array" in fun_name):
print("ENTRO")
var_len = c_int()
fun.argtypes.append(POINTER(c_int))
print("EY")
fun(c_char_p(section), c_char_p(var_name), byref(self._var),
byref(var_len))
print("len:", var_len.value)
#print (self._var[0])
#print (byref(var[0]))
a = ([self._var[i] for i in xrange(var_len.value)])
return a
else:
fun(c_char_p(section), c_char_p(var_name), byref(self._var))
return self._var.value
_py_finalize = _cpl_lib.CPLC_finalize
@abortMPI
def finalize(self):
self._py_finalize()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_setup_cfd = _cpl_lib.CPLC_setup_cfd
py_setup_cfd.argtypes = \
[c_int,
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def setup_cfd(self, icomm_grid, xyzL, xyz_orig, ncxyz):
"""
setup_cfd(icomm_grid, xyzL, xyz_orig, ncxyz):
"""
if (((type(icomm_grid) is list) and (len(icomm_grid) is 3)) or
((type(icomm_grid) is np.array) and (icomm_grid.shape[0] is 3))):
icomm_grid = self.COMM.Create_cart(
[icomm_grid[0], icomm_grid[1], icomm_grid[2]])
if ((type(xyzL) is list) or (xyzL.dtype != np.float64)
or (not xyzL.flags["F_CONTIGUOUS"])):
xyzL = np.array(xyzL, order='F', dtype=np.float64)
if ((type(xyz_orig) is list) or (xyz_orig.dtype != np.float64)
or (not xyz_orig.flags["F_CONTIGUOUS"])):
xyz_orig = np.array(xyz_orig, order='F', dtype=np.float64)
if ((type(ncxyz) is list) or (ncxyz.dtype != np.int32)
or (not ncxyz.flags["F_CONTIGUOUS"])):
ncxyz = np.array(ncxyz, order='F', dtype=np.int32)
self.py_setup_cfd(MPI._handleof(icomm_grid), xyzL, xyz_orig, ncxyz)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_setup_md = _cpl_lib.CPLC_setup_md
py_setup_md.argtypes = \
[c_int,
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def setup_md(self, icomm_grid, xyzL, xyz_orig):
"""
setup_md(icomm_grid, xyzL, xyz_orig)
"""
if (((type(icomm_grid) is list) and (len(icomm_grid) is 3)) or
((type(icomm_grid) is np.array) and (icomm_grid.shape[0] is 3))):
icomm_grid = self.COMM.Create_cart(
[icomm_grid[0], icomm_grid[1], icomm_grid[2]])
if ((type(xyzL) is list) or (xyzL.dtype != np.float64)
or (not xyzL.flags["F_CONTIGUOUS"])):
xyzL = np.array(xyzL, order='F', dtype=np.float64)
if ((type(xyz_orig) is list) or (xyz_orig.dtype != np.float64)
or (not xyz_orig.flags["F_CONTIGUOUS"])):
xyz_orig = np.array(xyz_orig, order='F', dtype=np.float64)
self.py_setup_md(MPI._handleof(icomm_grid), xyzL, xyz_orig)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_proc_extents = _cpl_lib.CPLC_proc_extents
py_proc_extents.argtypes = \
[ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'),
c_int,
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def proc_extents(self, coord, realm):
coord = self._type_check(coord)
extents = np.zeros(6, order='F', dtype=np.int32)
self.py_proc_extents(coord, realm, extents)
return extents
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_my_proc_extents = _cpl_lib.CPLC_my_proc_extents
py_my_proc_extents.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def my_proc_extents(self):
extents = np.zeros(6, order='F', dtype=np.int32)
self.py_my_proc_extents(extents)
return extents
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_proc_portion = _cpl_lib.CPLC_proc_portion
py_proc_portion.argtypes = \
[ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'),
c_int,
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def proc_portion(self, coord, realm, limits):
coord = self._type_check(coord)
limits = self._type_check(limits)
portion = np.zeros(6, order='F', dtype=np.int32)
self.py_proc_portion(coord, realm, limits, portion)
return portion
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_my_proc_portion = _cpl_lib.CPLC_my_proc_portion
py_my_proc_portion.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def my_proc_portion(self, limits):
limits = self._type_check(limits)
portion = np.zeros(6, order='F', dtype=np.int32)
self.py_my_proc_portion(limits, portion)
return portion
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_map_cfd2md_coord = _cpl_lib.CPLC_map_cfd2md_coord
py_map_cfd2md_coord.argtypes = \
[ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def map_cfd2md_coord(self, coord_cfd):
coord_cfd = self._type_check(coord_cfd)
coord_md = np.zeros(3, order='F', dtype=np.float64)
self.py_map_cfd2md_coord(coord_cfd, coord_md)
return coord_md
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_map_md2cfd_coord = _cpl_lib.CPLC_map_md2cfd_coord
py_map_md2cfd_coord.argtypes = \
[ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def map_md2cfd_coord(self, coord_md):
coord_md = self._type_check(coord_md)
coord_cfd = np.zeros(3, order='F', dtype=np.float64)
self.py_map_md2cfd_coord(coord_md, coord_cfd)
return coord_cfd
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_map_glob2loc_cell = _cpl_lib.CPLC_map_glob2loc_cell
py_map_glob2loc_cell.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def map_glob2loc_cell(self, limits, glob_cell):
limits = self._type_check(limits)
glob_cell = self._type_check(glob_cell)
loc_cell = np.zeros(3, order='F', dtype=np.int32)
self.py_map_glob2loc_cell(limits, glob_cell, loc_cell)
return loc_cell
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_map_cell2coord = _cpl_lib.CPLC_map_cell2coord
py_map_cell2coord.argtypes = \
[c_int, c_int, c_int,
ndpointer(np.float64, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def map_cell2coord(self, i, j, k):
coord = np.zeros(3, order='F', dtype=np.float64)
self.py_map_cell2coord(i, j, k, coord)
return coord
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_map_coord2cell = _cpl_lib.CPLC_map_coord2cell
py_map_coord2cell.argtypes = \
[c_double, c_double, c_double,
ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def map_coord2cell(self, x, y, z):
cell = np.zeros(3, order='F', dtype=np.int32)
self.py_map_coord2cell(x, y, z, cell)
return cell
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_get_no_cells = _cpl_lib.CPLC_get_no_cells
py_get_no_cells.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(3,), flags='aligned, f_contiguous')]
@abortMPI
def get_no_cells(self, limits):
limits = self._type_check(limits)
no_cells = np.zeros(3, order='F', dtype=np.int32)
self.py_get_no_cells(limits, no_cells)
return no_cells
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Limits of overlap region
py_get_olap_limits = _cpl_lib.CPLC_get_olap_limits
py_get_olap_limits.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def get_olap_limits(self):
limits = np.zeros(6, order='F', dtype=np.int32)
self.py_get_olap_limits(limits)
return limits
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Limits of contraint region
py_get_cnst_limits = _cpl_lib.CPLC_get_cnst_limits
py_get_cnst_limits.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def get_cnst_limits(self):
limits = np.zeros(6, order='F', dtype=np.int32)
self.py_get_cnst_limits(limits)
return limits
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Limits of boundary region
py_get_bnry_limits = _cpl_lib.CPLC_get_bnry_limits
py_get_bnry_limits.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def get_bnry_limits(self):
limits = np.zeros(6, order='F', dtype=np.int32)
self.py_get_bnry_limits(limits)
return limits
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_set_timing = _cpl_lib.CPLC_set_timing
py_set_timing.argtypes = \
[c_int, c_int, c_double]
#Don't call abortMPI so it can be handled nicely in Python.
#@abortMPI
def set_timing(self, initialstep, nsteps, dt):
class DepricatedException(Exception):
"""Raise Error as function should not be used"""
raise DepricatedException(
"CPL set_timing is depricated and should not be used")
self.py_set_timing(initialstep, nsteps, dt)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_send = _cpl_lib.CPLC_send
py_send.argtypes = \
[ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
POINTER(c_bool)]
@abortMPI
def send(self, asend, limits=None):
#Attempt to guess required size
if limits is None:
if self.realm is self.CFD_REALM:
limits = self.my_proc_portion(self.get_cnst_limits())
elif self.realm is self.MD_REALM:
limits = self.my_proc_portion(self.get_bnry_limits())
asend = self._type_check(asend)
asend_shape = np.array(asend.shape, order='F', dtype=np.int32)
send_flag = c_bool()
self.py_send(asend, asend_shape, limits, byref(send_flag))
return send_flag.value
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_recv = _cpl_lib.CPLC_recv
py_recv.argtypes = \
[ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
POINTER(c_bool)]
@abortMPI
def recv(self, arecv, limits=None):
#Attempt to guess required size
if limits is None:
if self.realm is self.CFD_REALM:
limits = self.my_proc_portion(self.get_bnry_limits())
elif self.realm is self.MD_REALM:
limits = self.my_proc_portion(self.get_cnst_limits())
arecv = self._type_check(arecv)
arecv_shape = np.array(arecv.shape, order='F', dtype=np.int32)
recv_flag = c_bool()
self.py_recv(arecv, arecv_shape, limits, byref(recv_flag))
return arecv, recv_flag.value
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_gather = _cpl_lib.CPLC_gather
py_gather.argtypes = \
[ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous')]
@abortMPI
def gather(self, gather_array, limits, recv_array):
gather_array = self._type_check(gather_array)
recv_array = self._type_check(recv_array)
gather_shape = np.array(gather_array.shape, order='F', dtype=np.int32)
recv_shape = np.array(recv_array.shape, order='F', dtype=np.int32)
self.py_gather(gather_array, gather_shape, limits, recv_array,
recv_shape)
return recv_array
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_scatter = _cpl_lib.CPLC_scatter
py_scatter.argtypes = \
[ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous'),
ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous'),
ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous')]
@abortMPI
def scatter(self, scatter_array, limits, recv_array):
scatter_array = self._type_check(scatter_array)
recv_array = self._type_check(recv_array)
scatter_shape = np.array(scatter_array.shape,
order='F',
dtype=np.int32)
recv_shape = np.array(recv_array.shape, order='F', dtype=np.int32)
self.py_scatter(scatter_array, scatter_shape, limits, recv_array,
recv_shape)
return recv_array
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_swaphalos = _cpl_lib.CPLC_swaphalos
py_swaphalos.argtypes = \
[ndpointer(np.float64, flags='aligned, f_contiguous'),
ndpointer(np.int32, ndim=1, flags='aligned, f_contiguous')]
@abortMPI
def swaphalos(self, A):
A = self._type_check(A)
A_shape = np.array(A.shape, order='F', dtype=np.int32)
self.py_swaphalos(A, A_shape)
return A
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_overlap = _cpl_lib.CPLC_overlap
py_overlap.argtypes = []
@abortMPI
def overlap(self):
self.py_overlap.restype = c_bool
return self.py_overlap()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
py_is_proc_inside = _cpl_lib.CPLC_is_proc_inside
py_is_proc_inside.argtypes = \
[ndpointer(np.int32, shape=(6,), flags='aligned, f_contiguous')]
@abortMPI
def is_proc_inside(self, region):
self.py_is_proc_inside.restype = c_bool
region = self._type_check(region)
return self.py_is_proc_inside(region)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def get(self, var_name):
try:
var_type = _CPL_GET_VARS[var_name]
fun = getattr(self._cpl_lib, "CPLC_" + var_name)
except KeyError:
print("CPL-ERROR: CPL Library function '" + str(var_name) +
"' not found!")
print("Available options include: ")
for var in _CPL_GET_VARS:
print(var)
raise KeyError
else:
fun.restype = var_type
return fun()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def set(self, var_name, value):
try:
var_type = _CPL_SET_VARS[var_name]
fun = getattr(self._cpl_lib, "CPLC_set_" + var_name)
except KeyError:
print("CPL-ERROR: CPL Library function '" + str(var_name) +
"' not found!")
raise KeyError
else:
fun.argtypes = [var_type]
return fun(var_type(value))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def _type_check(self, A):
if type(A) is list:
ndtype = type(A[0])
if ndtype == float:
ndtype = np.float64
elif ndtype == int:
ndtype = np.int32
A = np.asfortranarray(A, dtype=ndtype)
if not A.flags["F_CONTIGUOUS"]:
A = np.require(A, requirements=['F'])
if not A.flags["ALIGNED"]:
A = np.require(A, requirements=['A'])
return A
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def get_arrays(self, recv_size, send_size):
"""
Return recv array and send array based
on constraint/boundary sizes
"""
#Get constraint region
cnst_limits = self.get_cnst_limits()
cnst_portion = self.my_proc_portion(cnst_limits)
cnst_ncxl, cnst_ncyl, cnst_nczl = self.get_no_cells(cnst_portion)
#Get overlap region
BC_limits = self.get_bnry_limits()
BC_portion = self.my_proc_portion(BC_limits)
BC_ncxl, BC_ncyl, BC_nczl = self.get_no_cells(BC_portion)
#Allocate send and recv arrays
recv_array = np.zeros((recv_size, BC_ncxl, BC_ncyl, BC_nczl),
order='F',
dtype=np.float64)
send_array = np.zeros((send_size, cnst_ncxl, cnst_ncyl, cnst_nczl),
order='F',
dtype=np.float64)
return recv_array, send_array
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@abortMPI
def dump_region(self,
region,
array,
fname,
comm,
components={},
coords="mine"):
lines = ""
portion = self.my_proc_portion(region)
cell_coords = np.array(3)
dx = self.get("dx")
dy = self.get("dy")
dz = self.get("dz")
def_func = lambda x: x
components_dic = components
if callable(components):
def_func = components
if not components or callable(components):
components_idx = list(xrange(0, array.shape[0]))
for c_idx in components_idx:
components_dic[c_idx] = def_func
for k, v in components_dic.items():
if v is None:
components_dic[k] = def_func
#if self.overlap():
if self.is_proc_inside(portion):
ncx, ncy, ncz = self.get_no_cells(portion)
if (ncx, ncy, ncz) != array.shape[1:]:
print(
"self-Error in dump_region(): array and processor portion of different size."
)
MPI.COMM_WORLD.Abort(errorcode=1)
for i in xrange(portion[0], portion[1] + 1):
for j in xrange(portion[2], portion[3] + 1):
for k in xrange(portion[4], portion[5] + 1):
cell_coords = self.map_cell2coord(i, j, k)
if coords != "mine":
if self.realm == CPL.CFD_REALM:
cell_coords = self.map_cfd2md_coord(
cell_coords)
else:
cell_coords = self.map_md2cfd_coord(
cell_coords)
[i_loc, j_loc,
k_loc] = self.map_glob2loc_cell(portion, [i, j, k])
lines += str(cell_coords[0] + dx/2.0) + " "\
+ str(cell_coords[1] + dy/2.0) + " "\
+ str(cell_coords[2] + dz/2.0)
for k, f in components_dic.items():
lines += " " + str(f(array[k, i_loc, j_loc,
k_loc]))
lines += "\n"
# Gather all the forces from every processor and dump them to a file at the root
lines = comm.gather(lines, root=0)
myrank = comm.Get_rank()
if myrank == 0:
with open(fname, "w") as file_out:
file_out.writelines(lines)
from mpi4py import MPI
import cffi
import os
_libdir = os.path.dirname(__file__)
ffi = cffi.FFI()
if MPI._sizeof(MPI.Comm) == ffi.sizeof('int'):
_mpi_comm_t = 'int'
else:
_mpi_comm_t = 'void*'
ffi.cdef("""
typedef %(_mpi_comm_t)s MPI_Comm;
void sayhello(MPI_Comm);
""" % vars())
lib = ffi.dlopen(os.path.join(_libdir, "libhelloworld.so"))
def sayhello(comm):
comm_ptr = MPI._addressof(comm)
comm_val = ffi.cast('MPI_Comm*', comm_ptr)[0]
lib.sayhello(comm_val)
import ctypes
import numpy
import os
from ctypes import c_void_p, c_int, c_long, c_size_t, c_double
from mpi4py import MPI
# try to import the library
try:
path_to_library = os.path.join('lib','libhomework4.so')
homework4library = ctypes.cdll.LoadLibrary(path_to_library)
except OSError:
raise OSError("You need to compile your homework library using 'make'.")
# determine the c-type of an MPI_Comm type
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(c_int):
c_mpi_comm = ctypes.c_int
else:
c_mpi_comm = ctypes.c_void_p
###############################################################################
# function wrappers
###############################################################################
def heat_serial(u, dx, Nx, dt, num_steps):
r"""Solve the heat equation using basic finite difference scheme.
Parameters
----------
u : array
Function values.
from mpi4py import MPI
import ctypes
import os
_libdir = os.path.dirname(__file__)
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
_lib = ctypes.CDLL(os.path.join(_libdir, "libhelloworld.so"))
_lib.sayhello.restype = None
_lib.sayhello.argtypes = [MPI_Comm]
def sayhello(comm):
comm_ptr = MPI._addressof(comm)
comm_val = MPI_Comm.from_address(comm_ptr)
_lib.sayhello(comm_val)
from mpi4py import MPI
import cffi
import os
_libdir = os.path.dirname(__file__)
ffi = cffi.FFI()
if MPI._sizeof(MPI.Comm) == ffi.sizeof('int'):
_mpi_comm_t = 'int'
else:
_mpi_comm_t = 'void*'
ffi.cdef("""
typedef %(_mpi_comm_t)s MPI_Comm;
void sayhello(MPI_Comm);
""" % vars())
lib = ffi.dlopen(os.path.join(_libdir, "libhelloworld.so"))
def sayhello(comm):
comm_ptr = MPI._addressof(comm)
comm_val = ffi.cast('MPI_Comm*', comm_ptr)[0]
lib.sayhello(comm_val)
def testSizeOf(self):
for obj in self.objects:
n1 = MPI._sizeof(obj)
n2 = MPI._sizeof(type(obj))
self.assertEqual(n1, n2)
def init(self, comm: Optional[Union[Sequence[int], MPI.Comm]] = None,
process_sets: Optional[Sequence[ProcessSet]] = None):
"""A function that initializes Horovod.
Args:
comm: One of these possibilities:
1) List specifying ranks for the communicator, relative to the MPI_COMM_WORLD
communicator.
2) None: Use all ranks of MPI_COMM_WORLD.
3) MPI communicator to use. Given communicator will be duplicated and used as
the global Horovod communicator.
process_sets: One of these possibilities:
1) None -- Do not initialize any process sets.
2) List[hvd.ProcessSet] -- Initialize process set objects given in list (in addition to
hvd.global_process_set that will always be initialized). Users should hold on to these objects to pass
them to any Horovod collective communication ops. Duplicate process sets are not allowed.
3) "dynamic": do not initialize any process sets now, but set the environment variable
HOROVOD_DYNAMIC_PROCESS_SETS=1 so we can call `hvd.add_process_set(...)` later.
"""
if comm is None:
comm = []
if process_sets is None:
process_sets = []
elif isinstance(process_sets, str) and process_sets.lower() == "dynamic":
process_sets = []
os.environ["HOROVOD_DYNAMIC_PROCESS_SETS"] = "1"
process_sets = list(process_sets)
process_sets_via_ranks = [ps for ps in process_sets if ps.ranks is not None]
process_sets_via_comm = [ps for ps in process_sets if ps.mpi_comm is not None and ps.ranks is None]
process_set_sizes_via_ranks = [len(ps.ranks) for ps in process_sets_via_ranks]
process_set_ranks_via_ranks = [rank for process_set in process_sets_via_ranks for rank in process_set.ranks]
process_set_args_via_ranks = [
(ctypes.c_int * len(process_set_ranks_via_ranks))(*process_set_ranks_via_ranks),
(ctypes.c_int * len(process_set_sizes_via_ranks))(*process_set_sizes_via_ranks),
ctypes.c_int(len(process_set_sizes_via_ranks))
]
atexit.register(self.shutdown)
initialization_ok = True
if util.is_iterable(comm):
# comm is a list of ranks relative to the global communicator
if len(process_sets_via_comm) > 0:
raise NotImplementedError(
"At this time process sets defined via MPI communicators are only supported when calling hvd.init() "
"with comm set to a global MPI communicator.")
comm_size = len(comm)
initialization_ok = self.MPI_LIB_CTYPES.horovod_init(
(ctypes.c_int * comm_size)(*comm), ctypes.c_int(comm_size),
*process_set_args_via_ranks)
else:
if not self.mpi_built():
raise ValueError(
"Horovod has not been built with MPI support. Ensure MPI is installed and "
"reinstall Horovod with HOROVOD_WITH_MPI=1 to debug the build error.")
from mpi4py import MPI
if not isinstance(comm, MPI.Comm):
raise ValueError(
"Invalid type of argument comm. Expected list of rank integers or mpi4py.MPI.Comm object.")
global_process_set.mpi_comm = comm
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
comm_list = [comm] + [ps.mpi_comm for ps in process_sets_via_comm]
comm_objs = [MPI_Comm.from_address(MPI._addressof(c)) for c in comm_list]
num_comms = len(comm_list)
self.MPI_LIB_CTYPES.horovod_init_multi_comm.argtypes = [MPI_Comm * num_comms, ctypes.c_int]
initialization_ok = self.MPI_LIB_CTYPES.horovod_init_multi_comm((MPI_Comm * num_comms)(*comm_objs),
ctypes.c_int(num_comms),
*process_set_args_via_ranks)
if not initialization_ok:
raise ValueError(
"Horovod initialization failed. Please check log messages above for a more descriptive error.")
try:
_init_process_sets(process_sets)
except ValueError as e:
if (len(e.args) > 0 and isinstance(e.args[0], str) and
"Horovod has not been initialized properly" in e.args[0]):
# Horovod is already shutting down
return
else:
raise e
for ps_idx, ps in enumerate(process_sets):
if ps.process_set_id is None:
raise ValueError(
f"Horovod could not be initialized because process_sets entry number {ps_idx} is a duplicate: {ps}")
def __init__(self, library=None, style='spherical', dim=3, units='si', path=None, cmdargs=[], comm=None, ptr=None):
# What is ptr used for?
if library:
if not comm.Get_rank():
print("Using " + library + " as a shared library for DEM computations")
else:
if not comm.Get_rank():
print('No library supplies.')
sys.exit()
if not comm:
comm = MPI.COMM_WORLD
try:
self.lib = ctypes.CDLL(library, ctypes.RTLD_GLOBAL)
except:
etype,value,tb = sys.exc_info()
traceback.print_exception(etype,value,tb)
raise RuntimeError("Could not load LIGGGHTS dynamic library")
# if no ptr provided, create an instance of LIGGGHTS
# don't know how to pass an MPI communicator from PyPar
# but we can pass an MPI communicator from mpi4py v2.0.0 and later
# no_mpi call lets LIGGGHTS use MPI_COMM_WORLD
# cargs = array of C strings from args
# if ptr, then are embedding Python in LIGGGHTS input script
# ptr is the desired instance of LIGGGHTS
# just convert it to ctypes ptr and store in self.lmp
if MPI._sizeof(MPI.Comm) == ctypes.sizeof(ctypes.c_int):
MPI_Comm = ctypes.c_int
else:
MPI_Comm = ctypes.c_void_p
if not ptr:
# with mpi4py v2, can pass MPI communicator to LIGGGHTS
# need to adjust for type of MPI communicator object
# allow for int (like MPICH) or void* (like OpenMPI)
narg = 0
cargs = 0
if cmdargs:
cmdargs.insert(0, "liggghts.py")
narg = len(cmdargs)
for i in range(narg):
if isinstance(cmdargs[i], str):
cmdargs[i] = cmdargs[i].encode()
cargs = (ctypes.c_char_p*narg)(*cmdargs)
self.lib.lammps_open.argtypes = [ctypes.c_int, ctypes.c_char_p*narg, \
MPI_Comm, ctypes.c_void_p()]
else:
self.lib.lammps_open.argtypes = [ctypes.c_int, ctypes.c_int, \
MPI_Comm, ctypes.c_void_p()]
self.lib.lammps_open.restype = None
self.opened = 1
self.lmp = ctypes.c_void_p()
comm_ptr = MPI._addressof(comm)
comm_val = MPI_Comm.from_address(comm_ptr)
self.lib.lammps_open(narg,cargs,comm_val,ctypes.byref(self.lmp))
self.opened = True
else:
self.opened = False
if sys.version_info >= (3, 0):
# Python 3 (uses PyCapsule API)
ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]
self.lmp = ctypes.c_void_p(ctypes.pythonapi.PyCapsule_GetPointer(ptr, None))
else:
# Python 2 (uses PyCObject API)
ctypes.pythonapi.PyCObject_AsVoidPtr.restype = ctypes.c_void_p
ctypes.pythonapi.PyCObject_AsVoidPtr.argtypes = [ctypes.py_object]
self.lmp = ctypes.c_void_p(ctypes.pythonapi.PyCObject_AsVoidPtr(ptr))
def testSizeOf(self):
for obj in self.objects:
n1 = MPI._sizeof(obj)
n2 = MPI._sizeof(type(obj))
self.assertEqual(n1, n2)
from mpi4py import MPI
_conviqt = None
try:
_conviqt = ct.CDLL("_conviqt.so")
except OSError:
path = ctu.find_library("conviqt")
if path is not None:
_conviqt = ct.CDLL(path)
available = _conviqt is not None
if available:
try:
if MPI._sizeof(MPI.Comm) == ct.sizeof(ct.c_int):
MPI_Comm = ct.c_int
else:
MPI_Comm = ct.c_void_p
except Exception as e:
raise Exception(
'Failed to set the portable MPI communicator datatype: "{}". '
"MPI4py is probably too old. ".format(e))
def encode_comm(comm):
comm_ptr = MPI._addressof(comm)
return MPI_Comm.from_address(comm_ptr)
# Beam functions
_conviqt.conviqt_beam_new.restype = ct.c_void_p
class heffte_plan(Structure):
_fields_ = [("backend_type", c_int), ("using_r2c", c_int),
("fft", c_void_p)]
LP_plan = POINTER(heffte_plan)
class plan_options(Structure):
_fields_ = [("use_reorder", c_int), ("use_alltoall", c_int),
("use_pencils", c_int)]
# double-check this!
MPI_Comm = c_int if mpi._sizeof(mpi.COMM_WORLD) == sizeof(c_int) else c_void_p
libheffte = cdll.LoadLibrary(libheffte_path)
# create and destroy
libheffte.heffte_plan_create.argtypes = [c_int, ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
MPI_Comm, POINTER(plan_options), POINTER(LP_plan)]
libheffte.heffte_plan_create.restype = c_int
libheffte.heffte_plan_create_r2c.argtypes = [c_int, ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
ndpointer(c_int, flags="C_CONTIGUOUS"), ndpointer(c_int, flags="C_CONTIGUOUS"), \
c_int, MPI_Comm, POINTER(plan_options), POINTER(LP_plan)]
libheffte.heffte_plan_create_r2c.restype = c_int
