def test_bad_host(self):
logging.debug('')
logging.debug('test_bad_host')
if self.skip_ssh:
logging.debug(' requires ssh, skipping')
return
self.machines.append({'hostname': 'xyzzy', 'python': self.python})
self.cluster = ClusterAllocator(self.name, self.machines)
self.assertEqual(len(self.cluster), len(self.machines) - 1)
def test_bad_python(self):
logging.debug('')
logging.debug('test_bad_python')
if self.skip_ssh:
logging.debug(' requires ssh, skipping')
return
self.machines = [{'hostname': self.node, 'python': 'no-such-python'}]
self.cluster = ClusterAllocator(self.name, self.machines)
self.assertEqual(len(self.cluster), 0)
def init_cluster(encrypted=True, clean_dir=True, allow_shell=False):
"""
If not already done, initializes the ResourceAllocationManager and
adds a cluster using encrypted or unencrypted communication.
Returns the name of the configured cluster.
"""
authkey = 'PublicKey' if encrypted else 'AuthKey'
allocators = ResourceAllocationManager.list_allocators()
if len(allocators) == 1:
local = ResourceAllocationManager.get_allocator(0)
if local.max_load < 10: # First time we've been called.
# Ensure we aren't held up by local host load problems.
local.max_load = 10
if clean_dir:
# Remove any local allocator-created directories.
for path in glob.glob('Sim-*'):
shutil.rmtree(path, onerror=onerror)
node = platform.node()
name = '%s_%s' % (node.replace('.', '_'), authkey)
for allocator in allocators:
if allocator.name == name:
return name # Don't add multiple copies.
machines = []
python = sys.executable
if node.startswith('gxterm'):
# User environment assumed OK on this GRC cluster front-end.
# Using less than full machine (55 nodes) to allow multiple
# cluster testing without hitting limit on open files (sockets).
for i in range(20):
machines.append({'hostname': 'gx%02d' % i, 'python': python})
elif local_ssh_available():
machines.append({'hostname': node, 'python': python})
if machines:
cluster = ClusterAllocator(name, machines, authkey, allow_shell)
ResourceAllocationManager.insert_allocator(0, cluster)
return name
elif not encrypted:
# Create a LocalAllocator so we have *something*.
name = 'LocalUnencrypted'
for allocator in allocators:
if allocator.name == name:
return name # Don't add multiple copies.
local = LocalAllocator(name, authkey=authkey, allow_shell=allow_shell)
ResourceAllocationManager.insert_allocator(0, local)
return name
return None
def test_cluster(self):
logging.debug('')
logging.debug('test_cluster')
if self.skip_ssh:
logging.debug(' requires ssh, skipping')
return
self.cluster = ClusterAllocator(self.name, self.machines)
self.assertEqual(len(self.cluster), len(self.machines))
n_servers = self.cluster.max_servers(
{'python_version': sys.version[:3]})
try:
n_cpus = multiprocessing.cpu_count()
except (AttributeError, NotImplementedError): # pragma no cover
n_cpus = 1
self.assertEqual(n_servers, len(self.cluster) * n_cpus)
n_servers = self.cluster.max_servers({'python_version': 'bad-version'})
self.assertEqual(n_servers, 0)
def main():
""" Configure a cluster and use it. """
enable_console(logging.DEBUG)
logging.getLogger().setLevel(0)
print 'Client PID', os.getpid()
# Configure cluster.
cluster_name = 'EC2Cluster'
machines = []
if USE_EC2:
# The identity file used to access EC2 via ssh.
identity_filename = os.path.expanduser('~/.ssh/lovejoykey')
identity_filename += '.ppk' if sys.platform == 'win32' else '.pem'
machines.append(ClusterHost(
hostname='*****@*****.**',
python='setowns1_2013-05-06_09.17.04.529682' \
'/OpenMDAO-OpenMDAO-Framework-testbranch/devenv/bin/python',
tunnel_incoming=True, tunnel_outgoing=True,
identity_filename=identity_filename))
machines.append(ClusterHost(
hostname='*****@*****.**',
python='setowns1_2013-05-06_09.17.03.113077' \
'/OpenMDAO-OpenMDAO-Framework-testbranch/devenv/bin/python',
tunnel_incoming=True, tunnel_outgoing=True,
identity_filename=identity_filename))
machines.append(ClusterHost(
hostname='*****@*****.**',
python='setowns1_2013-05-06_09.17.05.434412' \
'/OpenMDAO-OpenMDAO-Framework-testbranch/devenv/bin/python',
tunnel_incoming=True, tunnel_outgoing=True,
identity_filename=identity_filename))
machines.append(ClusterHost(
hostname='*****@*****.**',
python='setowns1_2013-05-06_09.20.17.379627' \
'/OpenMDAO-OpenMDAO-Framework-testbranch/devenv/Scripts/python',
tunnel_incoming=True, tunnel_outgoing=True,
identity_filename=identity_filename))
machines.append(ClusterHost(
hostname='*****@*****.**',
python='setowns1_2013-05-06_09.19.49.348885' \
'/OpenMDAO-OpenMDAO-Framework-testbranch/devenv/Scripts/python',
tunnel_incoming=True, tunnel_outgoing=True,
identity_filename=identity_filename))
# machines.append(ClusterHost(
# hostname='viper.grc.nasa.gov',
# python='OpenMDAO-Framework/devenv/bin/python',
# tunnel_incoming=True, tunnel_outgoing=True,
# identity_filename=None))
else:
# Trivial local 'cluster' for debugging without remote host issues.
machines.append(
ClusterHost(hostname=socket.getfqdn(), python=sys.executable))
# machines.append(ClusterHost(
# hostname='viper.grc.nasa.gov',
# python='OpenMDAO-Framework/devenv/bin/python',
# tunnel_incoming=True, tunnel_outgoing=True,
# identity_filename=None))
# Start it.
cluster = ClusterAllocator(cluster_name,
machines,
allow_shell=True,
method='load-average')
# method='greedy')
# method='round-robin')
print 'Cluster initialized'
RAM.insert_allocator(0, cluster)
n_servers = RAM.max_servers(dict(allocator=cluster_name))
print n_servers, 'Servers:'
for name in RAM.get_hostnames(
dict(allocator=cluster_name, min_cpus=n_servers)):
print ' ', name
# Create model.
top = GPOptimization()
# Configure DOE.
top.driver.sequential = False # Run concurrently across cluster.
top.driver.reload_model = False
# Force use of only cluster hosts by adding this requirement.
top.driver.extra_resources = dict(allocator=cluster_name)
# This is necessary more often than it should be.
top.driver.ignore_egg_requirements = True
# Perform the optimization.
top.run()
