def initialize(parameterfile):
params = getparameters(parameterfile)
if params.serial: mpi4py.rc.initialize = False
from mpi4py import MPI
if MPI.Is_initialized():
params.comm = MPI.COMM_WORLD
params.rank = params.comm.Get_rank()
params.size = params.comm.Get_size()
params.parallel = True
else:
params.rank = 0
params.size = 1
params.parallel = False
fmt = '%H:%M:%S on %m/%d/%Y'
timestamp = datetime.datetime.now().strftime(fmt)
if (params.rank == 0):
print('')
bar = 72 * '-'
print(bar)
print('Running on', params.size, 'processor(s)')
print('Time: ' + timestamp)
print('Directory: ' + os.getcwd())
print(bar)
print('')
def __init__(self):
if not MPI.Is_initialized():
print("Manual MPI_Init performed.")
MPI.Init()
self.comm = MPI.COMM_WORLD
self.rank = self.comm.Get_rank()
self.size = self.comm.Get_size()
def _init_mpi():
"""provides a way to manually set the thread init mode for MPI if necessary.
Needs to happen as early as possible, otherwise mpi4py might auto-init somewhere else.
"""
try:
import mpi4py
except ImportError:
return
# only change finalize setting if unset
finalize = (mpi4py.rc.finalize is None) or mpi4py.rc.finalize
mpi4py.rc(initialize=False, finalize=finalize)
from mpi4py import MPI
if not MPI.Is_initialized():
required_level = int(
os.environ.get('PYMOR_MPI_INIT_THREAD', MPI.THREAD_MULTIPLE))
supported_lvl = MPI.Init_thread(required_level)
if supported_lvl < required_level:
print(
f'MPI does support threading level {required_level}, running with {supported_lvl} instead',
flush=True)
try:
# this solves sporadic mpi calls happening after finalize
import petsc4py
petsc4py.init()
except ImportError:
return
def __enter__(self):
try:
# Import MPI4Py without initializing it
import mpi4py.rc
mpi4py.rc.initialize = False
from mpi4py import MPI
if MPI.Is_initialized():
return "child"
except:
pass
n = self._n
if n <= 1:
return "child"
if os.getenv('D500_IN_MPI') is None:
env = os.environ.copy()
env.update(D500_IN_MPI="1")
# n = 2
args = ['mpiexec', '-np', str(n)]
args += [sys.executable] + sys.argv
print(' '.join(args))
subprocess.check_call(args, env=env)
return "parent"
else:
return "child"
def start_mpi(block_nonroot_stdout=True):
"""
Check if MPI has already been initialized. If so, just set the communicators,
Npus, and rank variables.
Parameters
----------
block_nonroot_stdout : bool (True)
Redirect stdout on nonzero ranks to /dev/null, for cleaner output.
"""
global world_comm, node_comm, rank_comm, rank, Npus
if not MPI.Is_initialized():
MPI.Init_thread(MPI.THREAD_MULTIPLE)
atexit.register(MPI.Finalize)
world_comm = MPI.COMM_WORLD
node_comm = world_comm.Split_type(MPI.COMM_TYPE_SHARED)
rank_comm = world_comm.Split(color=node_comm.rank)
Npus = world_comm.Get_size()
rank = world_comm.Get_rank()
set_mpi_excepthook(world_comm)
world_comm.Barrier()
if (not rank == 0) and block_nonroot_stdout: # pragma: no cover
# For non-root ranks, do not print to stdout.
# (Uncovered until we have multi-rank tests)
sys.stdout = open('/dev/null', 'w')
def report_collective_grid_sizes(ug):
from mpi4py import MPI
num_cells = ug.GetNumberOfCells()
mem_size = ug.GetActualMemorySize()
if MPI.Is_initialized():
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
total_cell_count = comm.reduce(num_cells, op=MPI.SUM)
max_cell_count = comm.reduce(num_cells, op=MPI.MAX)
min_cell_count = comm.reduce(num_cells, op=MPI.MIN)
total_mem_used = comm.reduce(mem_size, op=MPI.SUM)
min_mem_used = comm.reduce(mem_size, op=MPI.MIN)
max_mem_used = comm.reduce(mem_size, op=MPI.MAX)
if rank == 0:
print "Average grid cell count over MPI ranks: {:.1e}"\
.format(total_cell_count/float(size))
print "Minimum grid cell count over MPI ranks: {:.1e}"\
.format(min_cell_count)
print "Maximum grid cell count over MPI ranks: {:.1e}"\
.format(max_cell_count)
print "Average grid memory used over MPI ranks: {} MB"\
.format((total_mem_used/float(size))/1000.0)
print "Minimum grid memory used over MPI ranks: {} MB"\
.format(min_mem_used/1000.0)
print "Maximum grid memory used over MPI ranks: {} MB"\
.format(max_mem_used/1000.0)
def __init__(self,
src,
extension='pyx',
root=None,
verbose=False,
depends=None,
extra_inc_dirs=None,
extra_compile_args=None,
extra_link_args=None):
"""Initialize ExtModule.
Parameters
-----------
src : str : source code.
ext : str : extension for source code file.
Do not specify the '.' (defaults to 'pyx').
root : str: root of directory to store code and modules in.
If not set it defaults to "~/.cpy/source/<platform-directory>".
where <platform-directory> is platform specific.
verbose : Bool : Print messages for convenience.
depends : list : a list of modules that this extension depends on
if any of these have an m_time greater than the compiled extension
module, the extension will be recompiled.
extra_inc_dirs : list : a list of directories to look for .pxd, .h
and other files.
extra_compile_args: list : a list of extra compilation flags.
extra_link_args: list : a list of extra link flags.
"""
self._setup_root(root)
self.code = src
self.hash = get_md5(src)
self.extension = extension
self.name = 'm_{0}'.format(self.hash)
self._setup_filenames()
self.verbose = verbose
self.depends = depends
self.extra_inc_dirs = extra_inc_dirs if extra_inc_dirs else []
self.extra_compile_args = (extra_compile_args
if extra_compile_args else [])
self.extra_link_args = extra_link_args if extra_link_args else []
if MPI is not None and MPI.Is_initialized():
self.comm = MPI.COMM_WORLD
self.rank = self.comm.Get_rank()
self.num_procs = self.comm.Get_size()
else:
self.rank = 0
self.num_procs = 1
self.shared_filesystem = False
self._create_source()
def using_horovod():
"""
Returns true if the MPI environment is initialized, indicating that
`hvd.init()` has been called.
"""
if MPI is not None:
return MPI.Is_initialized()
return False
def test_init():
if not initialized():
initialize()
from mpi4py import MPI
assert initialized() == True
assert MPI.Is_initialized() == True
assert finalized() == False
assert MPI.Is_finalized() == False
def mpi_end_barrier():
""" Invokes a barrier and finalization if MPI is running, or nothing
otherwise. """
# Import MPI4Py without initializing it
import mpi4py.rc
mpi4py.rc.initialize = False
from mpi4py import MPI
if MPI.Is_initialized():
MPI.COMM_WORLD.Barrier()
MPI.Finalize()
def __init__(self):
from mpi4py import MPI
if MPI.Is_initialized():
self.comm = MPI.COMM_WORLD
self.rank = self.comm.Get_rank()
self.size = self.comm.Get_size()
self.start_time = time.time()
else:
self.size = 1
self.rank = 0
def __exit__(self, type, value, traceback):
try:
# Import MPI4Py without initializing it
import mpi4py.rc
mpi4py.rc.initialize = False
from mpi4py import MPI
mpi_init = MPI.Is_initialized()
except:
mpi_init = False
if mpi_init or os.getenv('D500_IN_MPI') is not None:
MPI.Finalize()
def __init__(self, shape, dimensions, input_comm=None, topology=None):
super(Distributor, self).__init__(shape, dimensions)
if configuration['mpi']:
# First time we enter here, we make sure MPI is initialized
if not MPI.Is_initialized():
MPI.Init()
global init_by_devito
init_by_devito = True
self._input_comm = (input_comm or MPI.COMM_WORLD).Clone()
# Make sure the cloned communicator will be freed up upon exit
def cleanup():
if self._input_comm is not None:
self._input_comm.Free()
atexit.register(cleanup)
if topology is None:
# `MPI.Compute_dims` sets the dimension sizes to be as close to each other
# as possible, using an appropriate divisibility algorithm. Thus, in 3D:
# * topology[0] >= topology[1] >= topology[2]
# * topology[0] * topology[1] * topology[2] == self._input_comm.size
# However, `MPI.Compute_dims` is distro-dependent, so we have to enforce
# some properties through our own wrapper (e.g., OpenMPI v3 does not
# guarantee that 9 ranks are arranged into a 3x3 grid when shape=(9, 9))
self._topology = compute_dims(self._input_comm.size,
len(shape))
else:
self._topology = topology
if self._input_comm is not input_comm:
# By default, Devito arranges processes into a cartesian topology.
# MPI works with numbered dimensions and follows the C row-major
# numbering of the ranks, i.e. in a 2x3 Cartesian topology (0,0)
# maps to rank 0, (0,1) maps to rank 1, (0,2) maps to rank 2, (1,0)
# maps to rank 3, and so on.
self._comm = self._input_comm.Create_cart(self._topology)
else:
self._comm = input_comm
else:
self._input_comm = None
self._comm = MPI.COMM_NULL
self._topology = tuple(1 for _ in range(len(shape)))
# The domain decomposition
self._decomposition = [
Decomposition(np.array_split(range(i), j), c)
for i, j, c in zip(shape, self.topology, self.mycoords)
]
def build(self, force=False):
"""Build source into an extension module. If force is False
previously compiled module is returned.
"""
if not self.shared_filesystem or self.rank == 0:
with self._lock():
if force or self.should_recompile():
self._message("Compiling code at:", self.src_path)
inc_dirs = [numpy.get_include()]
inc_dirs.extend(self.extra_inc_dirs)
extra_compile_args, extra_link_args = (
self._get_extra_args())
extension = Extension(
name=self.name,
sources=[self.src_path],
include_dirs=inc_dirs,
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
language="c++")
if not hasattr(sys.stdout, 'errors'):
# FIXME: This happens when nosetests replaces the
# stdout with the a Tee instance. This Tee instance
# does not have errors which breaks the tests so we
# disable verbose reporting.
script_args = []
else:
script_args = ['--verbose']
try:
with CaptureMultipleStreams() as stream:
mod = pyxbuild.pyx_to_dll(
self.src_path,
extension,
pyxbuild_dir=self.build_dir,
force_rebuild=True,
setup_args={'script_args': script_args})
except (CompileError, LinkError):
hline = "*" * 80
print(hline + "\nERROR")
print(stream.get_output()[0])
print(stream.get_output()[1])
msg = "Compilation of code failed, please check "\
"error messages above."
print(hline + "\n" + msg)
sys.exit(1)
shutil.copy(mod, self.ext_path)
else:
self._message("Precompiled code from:", self.src_path)
if MPI is not None and MPI.Is_initialized():
self.comm.barrier()
def onexit():
if not MPI.Is_initialized() or MPI.Is_finalized():
return
# Get the current exception (if any)
exc = excepthook.exception
# If we are exiting normally then call MPI_Finalize
if (MPI.COMM_WORLD.size == 1 or exc is None
or isinstance(exc, KeyboardInterrupt)
or (isinstance(exc, SystemExit) and exc.code == 0)):
MPI.Finalize()
# Otherwise forcefully abort
else:
MPI.COMM_WORLD.Abort(1)
def __new__(cls):
"""Creates a Serial WorkManager if size is 1. Otherwise creates a
single Manager and size-1 Worker.
"""
log.debug('MPIWorkManager.__new__()')
assert MPI.Is_initialized()
assert MPI.Is_thread_main()
rank = MPI.COMM_WORLD.Get_rank()
size = MPI.COMM_WORLD.Get_size()
if size == 1:
return super().__new__(Serial)
elif rank == 0:
return super().__new__(Manager)
else:
return super().__new__(Worker)
def __init__(
self,
run_function,
num_workers: int = None,
callbacks=None,
run_function_kwargs=None,
comm=None,
):
super().__init__(run_function, num_workers, callbacks,
run_function_kwargs)
if not MPI.Is_initialized():
MPI.Init_thread()
self.comm = comm if comm else MPI.COMM_WORLD
self.num_workers = self.comm.Get_size() - 1 # 1 rank is the master
self.sem = asyncio.Semaphore(self.num_workers)
logging.info(
f"Creating MPIPoolExecutor with {self.num_workers} max_workers...")
self.executor = MPIPoolExecutor(max_workers=self.num_workers)
logging.info("Creation of MPIPoolExecutor done")
def exit(self, status=0):
"""Exit the mpi4py processor with the given status.
@keyword status: The program exit status.
@type status: int
"""
# Execution on the slave.
if MPI.COMM_WORLD.rank != 0:
# Catch sys.exit being called on an executing slave.
if self.in_main_loop:
raise Exception('sys.exit unexpectedly called on slave!')
# Catch sys.exit
else:
sys.stderr.write('\n')
sys.stderr.write('***********************************************\n')
sys.stderr.write('\n')
sys.stderr.write('warning sys.exit called before mpi4py main loop\n')
sys.stderr.write('\n')
sys.stderr.write('***********************************************\n')
sys.stderr.write('\n')
MPI.COMM_WORLD.Abort()
# Execution on the master.
else:
# Slave clean up.
if MPI.Is_initialized() and not MPI.Is_finalized() and MPI.COMM_WORLD.rank == 0:
# Send the exit command to all slaves.
self._broadcast_command(Exit_command())
# Dump all results.
self._ditch_all_results()
# Exit the program with the given status.
sys.exit(status)
'never')
APOCALYPTIC = 'APOCALYPTIC'
# register new loglevelname
logging.addLevelName(logging.CRITICAL * 2 + 1, APOCALYPTIC)
# register QUIET, EXCEPTION and FATAL alias
logging._levelNames['EXCEPTION'] = logging.ERROR
logging._levelNames['FATAL'] = logging.CRITICAL
logging._levelNames['QUIET'] = logging.WARNING
# mpi rank support
_MPIRANK = MPIRANK_NO_MPI
if not _env_to_boolean('FANCYLOGGER_IGNORE_MPI4PY'):
try:
from mpi4py import MPI
if MPI.Is_initialized():
_MPIRANK = str(MPI.COMM_WORLD.Get_rank())
if MPI.COMM_WORLD.Get_size() > 1:
# enable mpi rank when mpi is used
FANCYLOG_FANCYRECORD = True
DEFAULT_LOGGING_FORMAT = DEFAULT_LOGGING_FORMAT_MPI
except ImportError:
pass
class MissingLevelName(KeyError):
pass
def getLevelInt(level_name):
"""Given a level name, return the int value"""
import os from mpi4py import rc assert rc.initialize is True assert rc.finalize is None assert rc.thread_level == 'multiple' os.environ['MPI4PY_RC_INITIALIZE'] = 'false' os.environ['MPI4PY_RC_FINALIZE'] = 'off' os.environ['MPI4PY_RC_THREAD_LEVEL'] = 'single' from mpi4py import MPI assert not MPI.Is_initialized() assert not MPI.Is_finalized() assert rc.initialize is False assert rc.finalize is False assert rc.thread_level == 'single'
def test_initialized_mpi4py(self):
# test mpi initialization (automatically when including mpi4py: https://mpi4py.readthedocs.io/en/stable/mpi4py.run.html)
self.assertTrue(mpi.Is_initialized())
def testIsInitialized(self):
flag = MPI.Is_initialized()
self.assertTrue(type(flag) is bool)
self.assertTrue(flag)
import os
import gmx.util
from gmx.util import to_string
from gmx.util import to_utf8
# # Get a test tpr filename
# from gmx.data import tpr_filename
# These tests will need to be updated when the workspec schema changes. These features appear in
# release 0.0.4, and schema changes that break these tests warrant a bump in workspec_version.
workspec_version = "gmxapi_workspec_0_1"
try:
from mpi4py import MPI
withmpi_only = pytest.mark.skipif(
not MPI.Is_initialized() or MPI.COMM_WORLD.Get_size() >= 2,
reason=
"Test requires at least 2 MPI ranks, but MPI is not initialized or too small."
)
except ImportError:
withmpi_only = pytest.mark.skip(
reason=
"Test requires at least 2 MPI ranks, but mpi4py is not available.")
# Some constants for this test module
file1 = "a.tpr"
file2 = "b.tpr"
class WorkSpecApiLevelTestCase(unittest.TestCase):
"""Make sure the tests match the module."""
import os
import shutil
import tempfile
import warnings
from contextlib import contextmanager
from enum import Enum
from typing import Union
import pytest
mpi_status = 'Test requires mpi4py managing 2 MPI ranks.'
skip_mpi = False
try:
from mpi4py import MPI
if not MPI.Is_initialized():
skip_mpi = True
mpi_status += ' MPI is not initialized'
elif MPI.COMM_WORLD.Get_size() < 2:
skip_mpi = True
mpi_status += ' MPI context is too small.'
except ImportError:
skip_mpi = True
mpi_status += ' mpi4py is not available.'
def pytest_configure(config):
config.addinivalue_line(
"markers", "withmpi_only: test requires mpi4py managing 2 MPI ranks.")
try:
import gmxapi as gmx
from gmxapi.simulation.context import Context as _context
from gmxapi.simulation.workflow import WorkElement, from_tpr
from gmxapi.version import api_is_at_least
except (ImportError, ModuleNotFoundError):
import gmx
from gmx import get_context as _context
from gmx.version import api_is_at_least
from gmx.workflow import from_tpr, WorkElement
nompi = lambda f: f
try:
from mpi4py import MPI
if MPI.Is_initialized():
rank = MPI.COMM_WORLD.Get_rank()
if MPI.COMM_WORLD.Get_size() > 1:
nompi = pytest.mark.skip(
reason='Test cannot run in a multirank MPI environment.')
else:
rank = 0
# Get a fixture for tests that should only run with 2 MPI ranks.
withmpi_only = pytest.mark.skipif(
not MPI.Is_initialized() or MPI.COMM_WORLD.Get_size() < 2,
reason=
"Test requires at least 2 MPI ranks, but MPI is not initialized or too small."
)
except (ImportError, ModuleNotFoundError):
withmpi_only = pytest.mark.skip(
def cleanup():
if MPI.Is_initialized():
MPI.Finalize()
def cleanup():
global init_by_devito
if init_by_devito and MPI.Is_initialized() and not MPI.Is_finalized():
MPI.Finalize()
def MDI_Init(arg1, arg2 = None):
global world_comm
global intra_code_comm
global use_mpi4py
global MPI
# attempt to import mpi4py
try:
import mpi4py
mpi4py.rc.initialize = False
from mpi4py import MPI
mpi4py.rc.initialize = True
if MPI.Is_initialized():
use_mpi4py = True
except ImportError:
pass
comm = None
if use_mpi4py:
comm = MPI.COMM_WORLD
# if this is a plugin code, get the plugin's MPI communicator
plugin_mode = MDI_Get_plugin_mode()
if ( plugin_mode == 1 and use_mpi4py ):
# Get a pointer to the C MPI communicator
python_plugin_mpi_world_ptr = MDI_Get_python_plugin_mpi_world_ptr()
# Convert the C MPI communicator to an MPI4Py communicator
c_mpi_communicator = ctypes.cast(python_plugin_mpi_world_ptr, ctypes.POINTER(ctypes.c_void_p)).contents.value
handle_t = ctypes.c_void_p
newobj = type(MPI.COMM_WORLD)()
handle_new = handle_t.from_address(MPI._addressof(newobj))
handle_new.value = c_mpi_communicator
__mdi_plugin_mpi_intra_comm__ = newobj
# Confirm that the new MPI communicator works
__mdi_plugin_mpi_intra_comm__.Get_size()
comm = __mdi_plugin_mpi_intra_comm__
# prepend the _language option, so that MDI knows this is a Python code
arg1 = "_language Python " + arg1
# determine the communication method
args = arg1.split()
mdi_method = None
for i in range(len(args)):
if args[i] == "-method" and i < len(args) - 1:
mdi_method = args[i+1]
if not mdi_method:
raise Exception("MDI Error: Unable to find -method option")
if mdi_method == "MPI":
# ensure that mpi4py is available
#if not use_mpi4py:
# raise Exception("MDI Error: When using the MPI communication method, mpi4py must be available")
# ensure that numpy is available
if use_mpi4py:
if not found_numpy:
raise Exception("MDI Error: When using the MPI communication method, numpy must be available")
if use_mpi4py:
world_comm = comm
intra_code_comm = comm
# send basic information about the MPI communicator to the MDI libarary
mpi_rank = comm.Get_rank()
mpi_world_size = comm.Get_size()
mdi.MDI_Set_World_Rank(mpi_rank)
mdi.MDI_Set_World_Size(mpi_world_size)
# set the MPI4Py callback functions
set_mpi4py_recv_callback()
set_mpi4py_send_callback()
set_mpi4py_size_callback()
set_mpi4py_rank_callback()
set_mpi4py_gather_names_callback()
set_mpi4py_barrier_callback()
set_mpi4py_split_callback()
# call MDI_Init
command = arg1.encode('utf-8')
ret = mdi.MDI_Init_with_options(ctypes.c_char_p(command) )
if ret != 0:
raise Exception("MDI Error: MDI_Init failed")
return ret
def spawn(self, **kwargs):
"""
Spawn MPI processes for and execute each of the managed targets.
Parameters
----------
kwargs: dict
options for the `info` argument in mpi spawn process.
see https://www.open-mpi.org/doc/v4.0/man3/MPI_Comm_spawn.3.php
"""
# Typcially MPI must be have intialized before spawning.
if not MPI.Is_initialized():
MPI.Init()
if self._is_parent:
# Find the path to the mpi_backend.py script (which should be in the
# same directory as this module:
parent_dir = os.path.dirname(__file__)
mpi_backend_path = os.path.join(parent_dir, 'mpi_backend.py')
# Set spawn option. Due to --oversubscribe, we will use none in binding
info = Info.Create()
info.Set('bind_to', "none")
for k, v in kwargs.items():
info.Set(k, v)
# Spawn processes:
self._intercomm = MPI.COMM_SELF.Spawn(sys.executable,
args=[mpi_backend_path],
maxprocs=len(self),
info=info)
# First, transmit twiggy logging emitters to spawned processes so
# that they can configure their logging facilities:
for i in self._targets:
self._intercomm.send(twiggy.emitters, i)
# Next, serialize the routing table ONCE and then transmit it to all
# of the child nodes:
try:
routing_table = self.routing_table
except:
routing_table = RoutingTable()
self.log_warning(
'Routing Table is null, using empty routing table.')
self._intercomm.bcast(routing_table, root=MPI.ROOT)
# Transmit class to instantiate, globals required by the class, and
# the constructor arguments; the backend will wait to receive
# them and then start running the targets on the appropriate nodes.
req = MPI.Request()
r_list = []
for i in self._targets:
target_globals = all_global_vars(self._targets[i])
# Serializing atexit with dill appears to fail in virtualenvs
# sometimes if atexit._exithandlers contains an unserializable function:
if 'atexit' in target_globals:
del target_globals['atexit']
data = (self._targets[i], target_globals, self._kwargs[i])
r_list.append(self._intercomm.isend(data, i))
# Need to clobber data to prevent all_global_vars from
# including it in its output:
del data
req.Waitall(r_list)
from mpi4py import rc rc.finalize = False from mpi4py import MPI assert MPI.Is_initialized() assert not MPI.Is_finalized() MPI.Finalize() assert MPI.Is_initialized() assert MPI.Is_finalized()
