def test_abbrev_nodenames_nochange_slurm():
env_resources = EnvResources()
# Test Slurm abbrev
exp_names = ['knl-0019', 'knl-0021', 'knl-0022', 'knl-0137', 'knl-0138', 'knl-0139', 'knl-2345']
env_resources.schedular = 'Slurm'
abbrev_names = env_resources.abbrev_nodenames(exp_names)
assert abbrev_names == exp_names, "Abbreviated names returned do not match expected"
del env_resources
def test_abbrev_nodenames_nochange_cobalt():
env_resources = EnvResources()
# Test Cobalt abbrev
exp_names = ['21', '22', '137', '138', '1234', '11234']
env_resources.schedular = 'Cobalt'
abbrev_names = env_resources.abbrev_nodenames(exp_names)
assert abbrev_names == exp_names, "Abbreviated names returned do not match expected"
del env_resources
def test_abbrev_nodenames_slurm():
env_resources = EnvResources()
# Test Slurm abbrev
exp_names = ['knl-0019', 'knl-0021', 'knl-0022']
full_names = ['knl-0019.some.suffix', 'knl-0021.some.suffix', 'knl-0022.diff_suffix']
env_resources.schedular = 'Slurm'
abbrev_names = env_resources.abbrev_nodenames(full_names)
assert abbrev_names == exp_names, "Abbreviated names returned do not match expected"
del env_resources
def test_shortnames_cobalt():
env_resources = EnvResources()
# Test Cobalt abbrev
exp_names = ['20', '21', '22', '137', '138', '1234', '11234']
full_names = [
'nid00020', 'nid00021', 'nid00022', 'nid00137', 'nid00138', 'nid01234',
'nid11234'
]
env_resources.scheduler = 'Cobalt'
abbrev_names = env_resources.shortnames(full_names)
assert abbrev_names == exp_names, "Abbreviated names returned do not match expected"
del env_resources
def test_get_global_nodelist_frm_wrklst_file():
# node_list file should override env variables
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "20-22,137-139,1234" # Should not be this
exp_out = ['knl-0019', 'knl-0021', 'knl-0022', 'knl-0137', 'knl-0138', 'knl-0139', 'knl-2345'] # Should be this
open('node_list', 'w').close()
try:
_ = Resources.get_global_nodelist(rundir=os.getcwd())
except ResourcesException as e:
assert e.args[0] == 'Error. global_nodelist is empty'
else:
assert 0
with open('node_list', 'w') as f:
for node in exp_out:
f.write(node + '\n')
# Do not specify env vars.
global_nodelist1 = Resources.get_global_nodelist(rundir=os.getcwd())
assert global_nodelist1 == exp_out, "global_nodelist returned does not match expected"
# Specify env vars - should ignore
env_resources = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="LIBE_RESOURCES_TEST_NODE_LIST",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
global_nodelist2 = Resources.get_global_nodelist(rundir=os.getcwd(), env_resources=env_resources)
assert global_nodelist2 == exp_out, "global_nodelist returned does not match expected"
os.remove('node_list')
def test_lsf_nodelist_shortform_seq():
os.environ[
"LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42 h21n18 42'
exp_out = ['g06n02', 'h21n18']
nodelist = EnvResources.get_lsf_nodelist_frm_shortform(
node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_get_global_nodelist_standalone():
mynode = socket.gethostname()
exp_node = mynode # sname(mynode)
env_resources = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(), env_resources=env_resources)
assert global_nodelist == [exp_node], "global_nodelist returned does not match expected"
def test_get_global_nodelist_frm_lsf_shortform():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42 h21n18 42'
exp_out = ['g06n02', 'h21n18']
env_resources = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="LIBE_RESOURCES_TEST_NODE_LIST")
global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(), env_resources=env_resources)
assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
def test_get_cpu_resources_from_env_unknown_env():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0009-0012]"
env_resources = EnvResources(nodelist_env_slurm="LIBE_RESOURCES_TEST_NODE_LIST",
nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
cores_info = node_resources._get_cpu_resources_from_env(env_resources=env_resources)
assert cores_info is None, 'cores_info should be None'
def test_get_cpu_resources_from_env_lsf_shortform():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42'
exp_out = (42, 42)
env_resources1 = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="LIBE_RESOURCES_TEST_NODE_LIST")
cores_info = node_resources._get_cpu_resources_from_env(env_resources=env_resources1)
assert cores_info == exp_out, "cores_info returned does not match expected"
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5 1 g06n02 42 h21n18 42'
env_resources2 = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="LIBE_RESOURCES_TEST_NODE_LIST")
cores_info = node_resources._get_cpu_resources_from_env(env_resources=env_resources2)
assert cores_info == exp_out, "cores_info returned does not match expected"
def test_get_global_nodelist_frm_cobalt():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "20-22,137-139,1234"
exp_out = ['20', '21', '22', '137', '138', '139', '1234']
env_resources = EnvResources(nodelist_env_slurm="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_cobalt="LIBE_RESOURCES_TEST_NODE_LIST",
nodelist_env_lsf="THIS_ENV_VARIABLE_IS_DEF_NOT_SET",
nodelist_env_lsf_shortform="THIS_ENV_VARIABLE_IS_DEF_NOT_SET")
global_nodelist = Resources.get_global_nodelist(rundir=os.getcwd(), env_resources=env_resources)
assert global_nodelist == exp_out, "global_nodelist returned does not match expected"
def test_slurm_nodelist_reverse_grp():
os.environ[
"LIBE_RESOURCES_TEST_NODE_LIST"] = "knl-[0020-0022,0139-0137,1234]"
exp_out = [
'knl-0020', 'knl-0021', 'knl-0022', 'knl-0137', 'knl-0138', 'knl-0139',
'knl-1234'
]
nodelist = EnvResources.get_slurm_nodelist(
node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_slurm_nodelist_groups_nodash():
os.environ[
"LIBE_RESOURCES_TEST_NODE_LIST"] = "nid0[0020-0022,0137-0139,1234]"
exp_out = [
'nid00020', 'nid00021', 'nid00022', 'nid00137', 'nid00138', 'nid00139',
'nid01234'
]
nodelist = EnvResources.get_slurm_nodelist(
node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_slurm_nodelist_groups_partitions():
os.environ[
"LIBE_RESOURCES_TEST_NODE_LIST"] = "bdw-[0254,0384,0565-0568],bdwd-[0004,0009]"
exp_out = [
'bdw-0254', 'bdw-0384', 'bdw-0565', 'bdw-0566', 'bdw-0567', 'bdw-0568',
'bdwd-0004', 'bdwd-0009'
]
nodelist = EnvResources.get_slurm_nodelist(
node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_slurm_nodelist_groups_longprefix():
os.environ[
"LIBE_RESOURCES_TEST_NODE_LIST"] = "super-[000020-000022,000137-000139,001234,023456-023458]"
exp_out = [
'super-000020', 'super-000021', 'super-000022', 'super-000137',
'super-000138', 'super-000139', 'super-001234', 'super-023456',
'super-023457', 'super-023458'
]
nodelist = EnvResources.get_slurm_nodelist(
node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_cobalt_nodelist_empty():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = ""
exp_out = [] # empty
nodelist = EnvResources.get_cobalt_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_slurm_nodelist_single_nodash():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "nid00056"
exp_out = ["nid00056"]
nodelist = EnvResources.get_slurm_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_slurm_nodelist_bdw_seq():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "bdw-[0009-0012]"
exp_out = ['bdw-0009', 'bdw-0010', 'bdw-0011', 'bdw-0012']
nodelist = EnvResources.get_slurm_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
class Resources:
"""Provides system resources to libEnsemble and executor.
This is intialized when the executor is created with auto_resources set to true.
**Object Attributes:**
These are set on initialization.
:ivar string top_level_dir: Directory where searches for node_list file
:ivar boolean central_mode: If true, then running in central mode; otherwise distributed
:ivar EnvResources env_resources: An object storing environment variables used by resources
:ivar list global_nodelist: A list of all nodes available for running user applications
:ivar int logical_cores_avail_per_node: Logical cores (including SMT threads) available on a node
:ivar int physical_cores_avail_per_node: Physical cores available on a node
:ivar WorkerResources worker_resources: An object that can contain worker specific resources
"""
DEFAULT_NODEFILE = 'node_list'
def __init__(self, top_level_dir=None,
central_mode=False,
allow_oversubscribe=False,
launcher=None,
cores_on_node=None,
node_file=None,
nodelist_env_slurm=None,
nodelist_env_cobalt=None,
nodelist_env_lsf=None,
nodelist_env_lsf_shortform=None):
"""Initializes a new Resources instance
Determines the compute resources available for current allocation, including
node list and cores/hardware threads available within nodes.
Parameters
----------
top_level_dir: string, optional
Directory libEnsemble runs in (default is current working directory)
central_mode: boolean, optional
If true, then running in central mode, otherwise distributed.
Central mode means libE processes (manager and workers) are grouped together and
do not share nodes with applications. Distributed mode means Workers share nodes
with applications.
allow_oversubscribe: boolean, optional
If false, then resources will raise an error if task process
counts exceed the CPUs available to the worker, as detected by
auto_resources. Larger node counts will always raise an error.
When auto_resources is off, this argument is ignored.
launcher: String, optional
The name of the job launcher, such as mpirun or aprun. This may be used to obtain
intranode information by launching a probing job onto the compute nodes.
If not present, the local node will be used to obtain this information.
cores_on_node: tuple (int,int), optional
If supplied gives (physical cores, logical cores) for the nodes. If not supplied,
this will be auto-detected.
node_file: String, optional
If supplied, give the name of a file in the run directory to use as a node-list
for use by libEnsemble. Defaults to a file named 'node_list'. If the file does
not exist, then the node-list will be auto-detected.
nodelist_env_slurm: String, optional
The environment variable giving a node list in Slurm format (Default: uses SLURM_NODELIST).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_cobalt: String, optional
The environment variable giving a node list in Cobalt format (Default: uses COBALT_PARTNAME).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_lsf: String, optional
The environment variable giving a node list in LSF format (Default: uses LSB_HOSTS).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_lsf_shortform: String, optional
The environment variable giving a node list in LSF short-form format (Default: uses LSB_MCPU_HOSTS)
Note: This is only queried if a node_list file is not provided and auto_resources=True.
"""
self.top_level_dir = top_level_dir or os.getcwd()
self.central_mode = central_mode
if self.central_mode:
logger.debug('Running in central mode')
self.allow_oversubscribe = allow_oversubscribe
self.env_resources = EnvResources(nodelist_env_slurm=nodelist_env_slurm,
nodelist_env_cobalt=nodelist_env_cobalt,
nodelist_env_lsf=nodelist_env_lsf,
nodelist_env_lsf_shortform=nodelist_env_lsf_shortform)
# This is global nodelist avail to workers - may change to global_worker_nodelist
if node_file is None:
node_file = Resources.DEFAULT_NODEFILE
self.global_nodelist = Resources.get_global_nodelist(node_file=node_file,
rundir=self.top_level_dir,
env_resources=self.env_resources)
self.launcher = launcher
remote_detect = False
if socket.gethostname() not in self.global_nodelist:
remote_detect = True
if not cores_on_node:
cores_on_node = \
node_resources.get_sub_node_resources(launcher=self.launcher,
remote_mode=remote_detect,
env_resources=self.env_resources)
self.physical_cores_avail_per_node = cores_on_node[0]
self.logical_cores_avail_per_node = cores_on_node[1]
self.libE_nodes = None
self.worker_resources = None
def add_comm_info(self, libE_nodes):
"""Adds comms-specific information to resources
Removes libEnsemble nodes from nodelist if in central_mode.
"""
self.libE_nodes = self.env_resources.abbrev_nodenames(libE_nodes)
libE_nodes_in_list = list(filter(lambda x: x in self.libE_nodes, self.global_nodelist))
if libE_nodes_in_list:
if self.central_mode and len(self.global_nodelist) > 1:
self.global_nodelist = Resources.remove_nodes(self.global_nodelist, self.libE_nodes)
if not self.global_nodelist:
logger.warning("Warning. Node-list for tasks is empty. Remove central_mode or add nodes")
def set_worker_resources(self, workerid, comm):
self.worker_resources = WorkerResources(workerid, comm, self)
@staticmethod
def get_MPI_variant():
"""Returns MPI base implementation
Returns
-------
mpi_variant: string:
MPI variant 'aprun' or 'jsrun' or 'mpich' or 'openmpi'
"""
try:
subprocess.check_call(['aprun', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return 'aprun'
except OSError:
pass
try:
subprocess.check_call(['jsrun', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return 'jsrun'
except OSError:
pass
try:
# Explore mpi4py.MPI.get_vendor() and mpi4py.MPI.Get_library_version() for mpi4py
try_mpich = subprocess.Popen(['mpirun', '-npernode'], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, _ = try_mpich.communicate()
if 'unrecognized argument npernode' in stdout.decode():
return 'mpich'
return 'openmpi'
except Exception:
pass
try:
subprocess.check_call(['srun', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return 'srun'
except OSError:
pass
# ---------------------------------------------------------------------------
# This is for central mode where libE nodes will not share with app nodes
@staticmethod
def remove_nodes(global_nodelist_in, remove_list):
"""Removes any nodes in remove_list from the global nodelist"""
global_nodelist = list(filter(lambda x: x not in remove_list, global_nodelist_in))
return global_nodelist
@staticmethod
def best_split(a, n):
"""Creates the most even split of list a into n parts and return list of lists"""
k, m = divmod(len(a), n)
return (a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))
@staticmethod
def get_global_nodelist(node_file=DEFAULT_NODEFILE,
rundir=None,
env_resources=None):
"""
Returns the list of nodes available to all libEnsemble workers.
If a node_file exists this is used, otherwise the environment
is interrogated for a node list. If a dedicated manager node is used,
then a node_file is recommended.
In central mode, any node with a libE worker is removed from the list.
"""
top_level_dir = rundir or os.getcwd()
node_filepath = os.path.join(top_level_dir, node_file)
global_nodelist = []
if os.path.isfile(node_filepath):
logger.debug("node_file found - getting nodelist from node_file")
with open(node_filepath, 'r') as f:
for line in f:
global_nodelist.append(line.rstrip())
else:
logger.debug("No node_file found - searching for nodelist in environment")
if env_resources:
global_nodelist = env_resources.get_nodelist()
if not global_nodelist:
# Assume a standalone machine
logger.info("Can not find nodelist from environment. Assuming standalone")
global_nodelist.append(socket.gethostname())
if global_nodelist:
return global_nodelist
raise ResourcesException("Error. global_nodelist is empty")
def __init__(self, top_level_dir=None,
central_mode=False,
allow_oversubscribe=False,
launcher=None,
cores_on_node=None,
node_file=None,
nodelist_env_slurm=None,
nodelist_env_cobalt=None,
nodelist_env_lsf=None,
nodelist_env_lsf_shortform=None):
"""Initializes a new Resources instance
Determines the compute resources available for current allocation, including
node list and cores/hardware threads available within nodes.
Parameters
----------
top_level_dir: string, optional
Directory libEnsemble runs in (default is current working directory)
central_mode: boolean, optional
If true, then running in central mode, otherwise distributed.
Central mode means libE processes (manager and workers) are grouped together and
do not share nodes with applications. Distributed mode means Workers share nodes
with applications.
allow_oversubscribe: boolean, optional
If false, then resources will raise an error if task process
counts exceed the CPUs available to the worker, as detected by
auto_resources. Larger node counts will always raise an error.
When auto_resources is off, this argument is ignored.
launcher: String, optional
The name of the job launcher, such as mpirun or aprun. This may be used to obtain
intranode information by launching a probing job onto the compute nodes.
If not present, the local node will be used to obtain this information.
cores_on_node: tuple (int,int), optional
If supplied gives (physical cores, logical cores) for the nodes. If not supplied,
this will be auto-detected.
node_file: String, optional
If supplied, give the name of a file in the run directory to use as a node-list
for use by libEnsemble. Defaults to a file named 'node_list'. If the file does
not exist, then the node-list will be auto-detected.
nodelist_env_slurm: String, optional
The environment variable giving a node list in Slurm format (Default: uses SLURM_NODELIST).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_cobalt: String, optional
The environment variable giving a node list in Cobalt format (Default: uses COBALT_PARTNAME).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_lsf: String, optional
The environment variable giving a node list in LSF format (Default: uses LSB_HOSTS).
Note: This is queried only if a node_list file is not provided and auto_resources=True.
nodelist_env_lsf_shortform: String, optional
The environment variable giving a node list in LSF short-form format (Default: uses LSB_MCPU_HOSTS)
Note: This is only queried if a node_list file is not provided and auto_resources=True.
"""
self.top_level_dir = top_level_dir or os.getcwd()
self.central_mode = central_mode
if self.central_mode:
logger.debug('Running in central mode')
self.allow_oversubscribe = allow_oversubscribe
self.env_resources = EnvResources(nodelist_env_slurm=nodelist_env_slurm,
nodelist_env_cobalt=nodelist_env_cobalt,
nodelist_env_lsf=nodelist_env_lsf,
nodelist_env_lsf_shortform=nodelist_env_lsf_shortform)
# This is global nodelist avail to workers - may change to global_worker_nodelist
if node_file is None:
node_file = Resources.DEFAULT_NODEFILE
self.global_nodelist = Resources.get_global_nodelist(node_file=node_file,
rundir=self.top_level_dir,
env_resources=self.env_resources)
self.launcher = launcher
remote_detect = False
if socket.gethostname() not in self.global_nodelist:
remote_detect = True
if not cores_on_node:
cores_on_node = \
node_resources.get_sub_node_resources(launcher=self.launcher,
remote_mode=remote_detect,
env_resources=self.env_resources)
self.physical_cores_avail_per_node = cores_on_node[0]
self.logical_cores_avail_per_node = cores_on_node[1]
self.libE_nodes = None
self.worker_resources = None
def test_cobalt_nodelist_seq():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "9-12"
exp_out = ['9', '10', '11', '12']
nodelist = EnvResources.get_cobalt_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_lsf_nodelist_single():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = 'batch5' + ' g06n02' * 42
exp_out = ["g06n02"]
nodelist = EnvResources.get_lsf_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"
def test_cobalt_nodelist_reverse_grp():
os.environ["LIBE_RESOURCES_TEST_NODE_LIST"] = "20-22,139-137,1234"
exp_out = ['20', '21', '22', '137', '138', '139', '1234']
nodelist = EnvResources.get_cobalt_nodelist(node_list_env="LIBE_RESOURCES_TEST_NODE_LIST")
assert nodelist == exp_out, "Nodelist returned does not match expected"