def enable_parallelism(suppress_logging=False, communicator=None):
"""
This method is used inside a script to turn on MPI parallelism, via
mpi4py. More information about running yt in parallel can be found
here: http://yt-project.org/docs/3.0/analyzing/parallel_computation.html
Parameters
----------
suppress_logging : bool
If set to True, only rank 0 will log information after the initial
setup of MPI.
communicator : mpi4py.MPI.Comm
The MPI communicator to use. This controls which processes yt can see.
If not specified, will be set to COMM_WORLD.
"""
global parallel_capable, MPI
try:
from mpi4py import MPI as _MPI
except ImportError:
mylog.info("mpi4py was not found. Disabling parallel computation")
parallel_capable = False
return
MPI = _MPI
exe_name = os.path.basename(sys.executable)
# if no communicator specified, set to COMM_WORLD
if communicator is None:
communicator = MPI.COMM_WORLD
parallel_capable = (communicator.size > 1)
if not parallel_capable: return False
mylog.info("Global parallel computation enabled: %s / %s",
communicator.rank, communicator.size)
communication_system.push(communicator)
ytcfg["yt","__global_parallel_rank"] = str(communicator.rank)
ytcfg["yt","__global_parallel_size"] = str(communicator.size)
ytcfg["yt","__parallel"] = "True"
if exe_name == "embed_enzo" or \
("_parallel" in dir(sys) and sys._parallel is True):
ytcfg["yt","inline"] = "True"
if communicator.rank > 0:
if ytcfg.getboolean("yt","LogFile"):
ytcfg["yt","LogFile"] = "False"
yt.utilities.logger.disable_file_logging()
yt.utilities.logger.uncolorize_logging()
# Even though the uncolorize function already resets the format string,
# we reset it again so that it includes the processor.
f = logging.Formatter("P%03i %s" % (communicator.rank,
yt.utilities.logger.ufstring))
if len(yt.utilities.logger.ytLogger.handlers) > 0:
yt.utilities.logger.ytLogger.handlers[0].setFormatter(f)
if ytcfg.getboolean("yt", "parallel_traceback"):
sys.excepthook = traceback_writer_hook("_%03i" % communicator.rank)
else:
sys.excepthook = default_mpi_excepthook
if ytcfg.getint("yt","LogLevel") < 20:
yt.utilities.logger.ytLogger.warning(
"Log Level is set low -- this could affect parallel performance!")
dtype_names.update(dict(
float32 = MPI.FLOAT,
float64 = MPI.DOUBLE,
int32 = MPI.INT,
int64 = MPI.LONG,
c = MPI.CHAR,
))
op_names.update(dict(
sum = MPI.SUM,
min = MPI.MIN,
max = MPI.MAX
))
# Turn off logging on all but the root rank, if specified.
if suppress_logging:
if communicator.rank > 0:
mylog.addFilter(FilterAllMessages())
return True
def enable_parallelism(suppress_logging: bool = False,
communicator=None) -> bool:
"""
This method is used inside a script to turn on MPI parallelism, via
mpi4py. More information about running yt in parallel can be found
here: https://yt-project.org/docs/3.0/analyzing/parallel_computation.html
Parameters
----------
suppress_logging : bool
If set to True, only rank 0 will log information after the initial
setup of MPI.
communicator : mpi4py.MPI.Comm
The MPI communicator to use. This controls which processes yt can see.
If not specified, will be set to COMM_WORLD.
Returns
-------
parallel_capable: bool
True if the call was successful. False otherwise.
"""
global parallel_capable, MPI
try:
from mpi4py import MPI as _MPI
except ImportError:
mylog.error("Could not enable parallelism: mpi4py is not installed")
return False
MPI = _MPI
exe_name = os.path.basename(sys.executable)
# if no communicator specified, set to COMM_WORLD
if communicator is None:
communicator = MPI.COMM_WORLD
parallel_capable = communicator.size > 1
if not parallel_capable:
mylog.error(
"Could not enable parallelism: only one mpi process is running. "
"To remedy this, launch the Python interpreter as\n"
" mpirun -n <X> python3 <yourscript>.py # with X > 1 ", )
return False
mylog.info(
"Global parallel computation enabled: %s / %s",
communicator.rank,
communicator.size,
)
communication_system.push(communicator)
ytcfg["yt", "internals", "global_parallel_rank"] = communicator.rank
ytcfg["yt", "internals", "global_parallel_size"] = communicator.size
ytcfg["yt", "internals", "parallel"] = True
if exe_name == "embed_enzo" or ("_parallel" in dir(sys)
and sys._parallel): # type: ignore
ytcfg["yt", "inline"] = True
yt.utilities.logger.uncolorize_logging()
# Even though the uncolorize function already resets the format string,
# we reset it again so that it includes the processor.
f = logging.Formatter("P%03i %s" %
(communicator.rank, yt.utilities.logger.ufstring))
if len(yt.utilities.logger.ytLogger.handlers) > 0:
yt.utilities.logger.ytLogger.handlers[0].setFormatter(f)
if ytcfg.get("yt", "parallel_traceback"):
sys.excepthook = traceback_writer_hook("_%03i" % communicator.rank)
else:
sys.excepthook = default_mpi_excepthook
if ytcfg.get("yt", "log_level") < 20:
yt.utilities.logger.ytLogger.warning(
"Log Level is set low -- this could affect parallel performance!")
dtype_names.update(
dict(
float32=MPI.FLOAT,
float64=MPI.DOUBLE,
int32=MPI.INT,
int64=MPI.LONG,
c=MPI.CHAR,
))
op_names.update(dict(sum=MPI.SUM, min=MPI.MIN, max=MPI.MAX))
# Turn off logging on all but the root rank, if specified.
if suppress_logging:
if communicator.rank > 0:
mylog.addFilter(FilterAllMessages())
return True
def enable_parallelism(suppress_logging=False, communicator=None):
"""
This method is used inside a script to turn on MPI parallelism, via
mpi4py. More information about running yt in parallel can be found
here: http://yt-project.org/docs/3.0/analyzing/parallel_computation.html
Parameters
----------
suppress_logging : bool
If set to True, only rank 0 will log information after the initial
setup of MPI.
communicator : mpi4py.MPI.Comm
The MPI communicator to use. This controls which processes yt can see.
If not specified, will be set to COMM_WORLD.
"""
global parallel_capable, MPI
try:
from mpi4py import MPI as _MPI
except ImportError:
mylog.info("mpi4py was not found. Disabling parallel computation")
parallel_capable = False
return
MPI = _MPI
exe_name = os.path.basename(sys.executable)
# if no communicator specified, set to COMM_WORLD
if communicator is None:
communicator = MPI.COMM_WORLD
parallel_capable = (communicator.size > 1)
if not parallel_capable: return False
mylog.info("Global parallel computation enabled: %s / %s",
communicator.rank, communicator.size)
communication_system.push(communicator)
ytcfg["yt", "__global_parallel_rank"] = str(communicator.rank)
ytcfg["yt", "__global_parallel_size"] = str(communicator.size)
ytcfg["yt", "__parallel"] = "True"
if exe_name == "embed_enzo" or \
("_parallel" in dir(sys) and sys._parallel is True):
ytcfg["yt", "inline"] = "True"
if communicator.rank > 0:
if ytcfg.getboolean("yt", "LogFile"):
ytcfg["yt", "LogFile"] = "False"
yt.utilities.logger.disable_file_logging()
yt.utilities.logger.uncolorize_logging()
# Even though the uncolorize function already resets the format string,
# we reset it again so that it includes the processor.
f = logging.Formatter("P%03i %s" %
(communicator.rank, yt.utilities.logger.ufstring))
if len(yt.utilities.logger.ytLogger.handlers) > 0:
yt.utilities.logger.ytLogger.handlers[0].setFormatter(f)
if ytcfg.getboolean("yt", "parallel_traceback"):
sys.excepthook = traceback_writer_hook("_%03i" % communicator.rank)
else:
sys.excepthook = default_mpi_excepthook
if ytcfg.getint("yt", "LogLevel") < 20:
yt.utilities.logger.ytLogger.warning(
"Log Level is set low -- this could affect parallel performance!")
dtype_names.update(
dict(
float32=MPI.FLOAT,
float64=MPI.DOUBLE,
int32=MPI.INT,
int64=MPI.LONG,
c=MPI.CHAR,
))
op_names.update(dict(sum=MPI.SUM, min=MPI.MIN, max=MPI.MAX))
# Turn off logging on all but the root rank, if specified.
if suppress_logging:
if communicator.rank > 0:
mylog.addFilter(FilterAllMessages())
return True
