def execute(self):
storage = Configurator().create_cluster_storage()
cluster_names = storage.get_stored_clusters()
if not cluster_names:
print("No clusters found.")
else:
print("""
The following clusters have been started.
Please note that there's no guarantee that they are fully configured:
""")
for name in sorted(cluster_names):
try:
cluster = Configurator().load_cluster(name)
except ConfigurationError, ex:
log.error("gettin information from cluster `%s`: %s",
name, ex)
continue
print("%s " % name)
print("-"*len(name))
print(" name: %s" % cluster.name)
print(" template: %s" % cluster.template)
print(" cloud: %s " % cluster._cloud)
for cls in cluster.nodes:
print(" - %s nodes: %d" % (cls, len(cluster.nodes[cls])))
print("")
def test_create_node(self):
configurator = Configurator(self.config)
node = configurator.create_node("mycluster", "compute", None, "test-1")
self.assertTrue(type(node) is Node)
self.assertEqual(node.name, "test-1")
self.assertEqual(node.type, "compute")
self.assertEqual(node._cloud_provider, None)
pub_key = self.config['mycluster']['login']['user_key_public']
self.assertEqual(node.user_key_public, pub_key)
prv_key = self.config['mycluster']['login']['user_key_private']
self.assertEqual(node.user_key_private, prv_key)
key_name = self.config['mycluster']['login']['user_key_name']
self.assertEqual(node.user_key_name, key_name)
usr = self.config['mycluster']['login']['image_user']
self.assertEqual(node.image_user, usr)
nodes = self.config['mycluster']['nodes']
sec_group = nodes['compute']['security_group']
self.assertEqual(node.security_group, sec_group)
image = self.config['mycluster']['nodes']['compute']['image_id']
self.assertEqual(node.image, image)
flavor = self.config['mycluster']['nodes']['compute']['flavor']
self.assertEqual(node.flavor, flavor)
self.assertEqual(node.image_userdata, '')
def test_create_node(self):
configurator = Configurator()
cloud_provider = BotoCloudProvider(config_cloud_ec2_url, config_cloud_ec2_region, config_cloud_ec2_access_key, config_cloud_ec2_secret_key)
node_frontend = configurator.create_node(config_cluster_name, Node.frontend_type, cloud_provider, "node001")
assert node_frontend
assert isinstance(node_frontend, Node)
assert node_frontend.name == "node001"
assert node_frontend.type == Node.frontend_type
assert node_frontend._cloud_provider == cloud_provider
assert node_frontend.user_key_public == os.path.expanduser(config_login_user_key_public)
assert node_frontend.user_key_name == os.path.expanduser(config_login_user_key_name)
assert node_frontend.image_user == config_login_image_user
assert node_frontend.security_group == config_frontend_security_group
assert node_frontend.image == config_frontend_image
assert node_frontend.image_userdata == config_frontend_image_userdata
assert node_frontend.flavor == config_frontend_flavor
node_compute = configurator.create_node(config_cluster_name, Node.compute_type, cloud_provider, "node001")
assert node_compute
assert isinstance(node_compute, Node)
assert node_compute.name == "node001"
assert node_compute.type == Node.compute_type
assert node_compute._cloud_provider == cloud_provider
assert node_compute.user_key_public == os.path.expanduser(config_login_user_key_public)
assert node_compute.user_key_name == config_login_user_key_name
assert node_compute.image_user == config_login_image_user
assert node_compute.security_group == config_compute_security_group
assert node_compute.image == config_compute_image
assert node_compute.image_userdata == config_compute_image_userdata
assert node_compute.flavor == config_compute_flavor
def execute(self):
Configuration.Instance().cluster_name = self.params.cluster
cluster_name = self.params.cluster
try:
cluster = Configurator().load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Setting up cluster %s: %s\n" %
(cluster_name, ex))
return
def execute(self):
# Get current cluster configuration
cluster_name = self.params.cluster
try:
cluster = Configurator().load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def test_create_cloud_provider(self):
configurator = Configurator()
cloud_provider = configurator.create_cloud_provider(config_cloud_name)
assert cloud_provider
assert isinstance(cloud_provider, AbstractCloudProvider)
assert cloud_provider._url == config_cloud_ec2_url
assert cloud_provider._region_name == config_cloud_ec2_region
assert cloud_provider._access_key == config_cloud_ec2_access_key
assert cloud_provider._secret_key == config_cloud_ec2_secret_key
def test_create_cluster_storage(self):
# default storage path
configurator = Configurator(self.config)
storage = configurator.create_cluster_storage()
default_storage = configurator.general_conf['storage']
self.assertEqual(storage._storage_dir, default_storage)
# custom storage path
path = "/tmp"
configurator = Configurator(self.config, storage_path=path)
storage = configurator.create_cluster_storage()
self.assertEqual(storage._storage_dir, path)
def test_create_cluster(self):
configurator = Configurator()
cluster = configurator.create_cluster(config_cluster_name)
assert cluster
assert isinstance(cluster, Cluster)
assert cluster.name == config_cluster_name
assert cluster._cloud == config_cluster_cloud
assert isinstance(cluster._cloud_provider, AbstractCloudProvider)
assert cluster._frontend == int(config_cluster_frontend)
assert cluster._compute == int(config_cluster_compute)
assert isinstance(cluster._setup_provider, AbstractSetupProvider)
def test_create_cluster(self):
configurator = Configurator(self.config)
cluster = configurator.create_cluster("mycluster")
self.assertEqual(cluster.name, "mycluster")
self.assertTrue(type(cluster._cloud_provider) is BotoCloudProvider)
self.assertTrue(type(cluster._setup_provider) is AnsibleSetupProvider)
self.assertTrue("compute" in cluster.nodes)
self.assertTrue("frontend" in cluster.nodes)
self.assertTrue(len(cluster.nodes["compute"]) == 2)
self.assertTrue(len(cluster.nodes["frontend"]) == 1)
def execute(self):
"""
Lists all nodes within the specified cluster with certain
information like id and ip.
"""
Configuration.Instance().cluster_name = self.params.cluster
cluster_name = self.params.cluster
try:
cluster = Configurator().load_cluster(cluster_name)
if self.params.update:
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def test_create_cloud_provider(self):
configurator = Configurator(self.config)
provider = configurator.create_cloud_provider("mycluster")
url = self.config['mycluster']['cloud']['ec2_url']
self.assertEqual(provider._url, url)
access_key = self.config['mycluster']['cloud']['ec2_access_key']
self.assertEqual(provider._access_key, access_key)
secret_key = self.config['mycluster']['cloud']['ec2_secret_key']
self.assertEqual(provider._secret_key, secret_key)
region = self.config['mycluster']['cloud']['ec2_region']
self.assertEqual(provider._region_name, region)
def _test_multiple_config_files(self, *paths_and_configs):
"""
Common code for all `test_read_multiple_config*` checks.
"""
tmpdir = None
try:
tmpdir = tempfile.mkdtemp()
paths = []
for path, content in paths_and_configs:
path = os.path.join(tmpdir, path)
basedir = os.path.dirname(path)
if not os.path.exists(basedir):
os.makedirs(basedir)
with open(path, 'w') as cfgfile:
cfgfile.write(content)
paths.append(path)
config = Configurator.fromConfig(paths)
# check all clusters are there
cfg = config.cluster_conf
self.assertTrue("slurm" in cfg)
# check for nodes
self.assertTrue("frontend" in cfg["slurm"]["nodes"])
self.assertTrue("compute" in cfg["slurm"]["nodes"])
# check one property in each category
self.assertEqual(cfg["slurm"]["cluster"]["security_group"], "default")
self.assertEqual(cfg["slurm"]["login"]["image_user"], "ubuntu")
self.assertEqual(cfg["slurm"]["setup"]["provider"], "ansible")
self.assertEqual(cfg["slurm"]["cloud"]["ec2_region"], "nova")
finally:
if tmpdir:
shutil.rmtree(tmpdir)
def test_setup_provider_using_environment(self):
config = copy.deepcopy(self.config)
configurator = Configurator(config)
# Save current variable, modify it and check if it's correctly read
SAVEDUSERNAME=os.getenv('OS_USERNAME')
os.environ['OS_USERNAME'] = '******'
provider = configurator.create_cloud_provider("os-cluster")
try:
self.assertEqual(provider._os_username, 'newusername')
except:
if SAVEDUSERNAME:
os.environ['OS_USERNAME'] = SAVEDUSERNAME
else:
del os.environ['OS_USERNAME']
raise
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
config = configurator.cluster_conf
print("""%d cluster templates found in configuration file.""" % len(config))
templates = config.keys()
for pattern in self.params.clusters:
templates = [t for t in templates if fnmatch(t, pattern)]
if self.params.clusters:
print("""%d cluter templates found matching pattern(s) '%s'""" % (len(templates), str.join(", ", self.params.clusters)))
for template in templates:
try:
cluster = configurator.create_cluster(template, template)
print("""
name: %s""" % template)
for nodekind in cluster.nodes:
print("%s nodes: %d" % (
nodekind,
len(cluster.nodes[nodekind])))
except ConfigurationError, ex:
log.warning("unable to load cluster `%s`: %s", template, ex)
def execute(self):
"""
Starts a new cluster.
"""
cluster_template = self.params.cluster
if self.params.cluster_name:
cluster_name = self.params.cluster_name
else:
cluster_name = self.params.cluster
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
# overwrite configuration
for option, value in self.params.extra_conf.iteritems():
cconf = configurator.cluster_conf[cluster_template]['cluster']
if option in cconf:
cconf[option] = value
# First, check if the cluster is already created.
try:
cluster = configurator.load_cluster(cluster_name)
except ClusterNotFound as e:
try:
cluster = configurator.create_cluster(
cluster_template, cluster_name)
except ConfigurationError, e:
log.error("Starting cluster %s: %s\n" % (cluster_template, e))
return
def test_load_cluster(self):
# test without storage file
storage_path = tempfile.mkdtemp()
configurator = Configurator(self.config, storage_path=storage_path)
self.assertRaises(ClusterNotFound,
configurator.load_cluster, "mycluster")
shutil.rmtree(storage_path)
def test_create_cluster_with_nodes_min(self):
cfg = self.config.copy()
cfg['mycluster']['cluster']['compute_nodes_min'] = 1
configurator = Configurator(cfg)
cconf = configurator.cluster_conf['mycluster']['cluster']
self.assertEqual(cconf['compute_nodes_min'], 1)
def get_configurator(configfiles='~/.elasticluster/config',
storage_path=None,
include_config_dirs=True):
conf = Configurator.fromConfig(configfiles,
storage_path,
include_config_dirs=True)
return conf
def ecluster_config(econfig_file, name=None):
"""Load the Elasticluster configuration."""
config = Configurator.fromConfig(econfig_file, get_storage_dir(econfig_file))
if not name:
return config
if name not in config.cluster_conf:
raise Exception('Cluster {} is not defined in {}.\n'.format(
name, os.path.expanduser(econfig_file)))
return config.cluster_conf[name]
def test_create_cluster(self):
configurator = Configurator(self.config)
cluster = configurator.create_cluster("mycluster")
self.assertEqual(cluster.template, "mycluster")
self.assertEqual(cluster.name, "mycluster")
cloud = self.config['mycluster']['cluster']['cloud']
self.assertEqual(cluster._cloud, cloud)
self.assertTrue(type(cluster._cloud_provider) is BotoCloudProvider)
self.assertTrue(type(cluster._setup_provider) is AnsibleSetupProvider)
self.assertTrue("compute" in cluster.nodes)
self.assertTrue("frontend" in cluster.nodes)
self.assertTrue(len(cluster.nodes["compute"]) == 2)
self.assertTrue(len(cluster.nodes["frontend"]) == 1)
def execute(self):
configurator = Configurator.fromConfig(self.params.config, storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Setting up cluster %s: %s\n" % (cluster_name, ex))
return
def missing_option(section, option):
_, cfgfile = tempfile.mkstemp()
cfg.remove_option(section, option)
with open(cfgfile, 'w') as fd:
cfg.write(fd)
try:
config = Configurator.fromConfig(cfgfile)
finally:
os.unlink(cfgfile)
def test_default_storage_options(self):
cfg = minimal_configuration(self.cfgfile)
with open(self.cfgfile, 'w') as fd:
cfg.write(fd)
config = Configurator.fromConfig(self.cfgfile)
repo = config.create_repository()
self.assertEqual(repo.storage_path, Configurator.default_storage_path)
self.assertEqual(repo.default_store.file_ending, Configurator.default_storage_type)
def missing_option(section, option):
_, cfgfile = tempfile.mkstemp()
cfg.remove_option(section, option)
with open(cfgfile, 'w') as fd:
cfg.write(fd)
try:
config = Configurator.fromConfig(cfgfile)
finally:
os.unlink(cfgfile)
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Setting up cluster %s: %s\n" % (cluster_name, ex))
return
def ecluster_config(econfig_file, name=None):
"""Load the Elasticluster configuration."""
config = Configurator.fromConfig(econfig_file,
get_storage_dir(econfig_file))
if not name:
return config
if name not in config.cluster_conf:
raise Exception('Cluster {} is not defined in {}.\n'.format(
name, os.path.expanduser(econfig_file)))
return config.cluster_conf[name]
def test_create_setup_provider(self):
configurator = Configurator(self.config)
provider = configurator.create_setup_provider("mycluster")
self.assertTrue(type(provider) is AnsibleSetupProvider)
prv_key = self.config['mycluster']['login']['user_key_private']
self.assertEqual(provider._private_key_file, prv_key)
usr = self.config['mycluster']['login']['image_user']
self.assertEqual(provider._remote_user, usr)
usr_sudo = self.config['mycluster']['login']['image_user_sudo']
self.assertEqual(provider._sudo_user, usr_sudo)
sudo = self.config['mycluster']['login']['image_sudo']
self.assertEqual(provider._sudo, sudo)
pb = self.config['mycluster']['setup']['playbook_path']
self.assertEqual(provider._playbook_path, pb)
def execute(self):
"""
Stops the cluster if it's running.
"""
cluster_name = self.params.cluster
configurator = Configurator.fromConfig(self.params.config, storage_path=self.params.storage)
try:
cluster = configurator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError), ex:
log.error("Stopping cluster %s: %s\n" % (cluster_name, ex))
return
def missing_option(section, option):
with pytest.raises(Invalid, MultipleInvalid):
tmpcfg = minimal_configuration()
_, cfgfile = tempfile.mkstemp()
tmpcfg.remove_option(section, option)
with open(cfgfile, 'w') as fd:
tmpcfg.write(fd)
try:
config = Configurator.fromConfig(cfgfile)
finally:
os.unlink(cfgfile)
def test_create_setup_provider(self):
configurator = Configurator(self.config)
provider = configurator.create_setup_provider("mycluster")
self.assertTrue(type(provider) is AnsibleSetupProvider)
prv_key = self.config['mycluster']['login']['user_key_private']
self.assertEqual(provider._private_key_file, prv_key)
usr = self.config['mycluster']['login']['image_user']
self.assertEqual(provider._remote_user, usr)
usr_sudo = self.config['mycluster']['login']['image_user_sudo']
self.assertEqual(provider._sudo_user, usr_sudo)
sudo = self.config['mycluster']['login']['image_sudo']
self.assertEqual(provider._sudo, sudo)
pb = self.config['mycluster']['setup']['playbook_path']
self.assertEqual(provider._playbook_path, pb)
def missing_option(section, option):
with pytest.raises(Invalid, MultipleInvalid):
tmpcfg = minimal_configuration()
_, cfgfile = tempfile.mkstemp()
tmpcfg.remove_option(section, option)
with open(cfgfile, 'w') as fd:
tmpcfg.write(fd)
try:
config = Configurator.fromConfig(cfgfile)
finally:
os.unlink(cfgfile)
def test_custom_storage_options(self):
cfg = minimal_configuration(self.cfgfile)
cfg.add_section('storage')
cfg.set('storage', 'storage_path', '/foo/bar')
cfg.set('storage', 'storage_type', 'json')
with open(self.cfgfile, 'w') as fd:
cfg.write(fd)
config = Configurator.fromConfig(self.cfgfile)
repo = config.create_repository()
self.assertEqual(repo.storage_path, '/foo/bar')
self.assertEqual(repo.default_store.file_ending, 'json')
def execute(self):
"""
Stops the cluster if it's running.
"""
cluster_name = self.params.cluster
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
try:
cluster = configurator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError), ex:
log.error("Stopping cluster %s: %s\n" % (cluster_name, ex))
return
def execute(self):
configurator = Configurator.fromConfig(self.params.config, storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
template = self.params.template
try:
cluster = configurator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" % (cluster_name, ex))
return
def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage,
include_config_dirs=True)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def test_create_setup_provider(self):
configurator = Configurator(self.config)
provider = configurator.create_setup_provider("mycluster")
self.assertTrue(type(provider) is AnsibleSetupProvider)
conf = self.config['mycluster']['setup']
groups = dict((k[:-7], v.split(',')) for k, v
in conf.items() if k.endswith('_groups'))
self.assertEqual(groups, provider.groups)
playbook_path = os.path.join(sys.prefix,
'share/elasticluster/providers/ansible-playbooks', 'site.yml')
self.assertEqual(playbook_path, provider._playbook_path)
storage_path = configurator.general_conf['storage']
self.assertEqual(provider._storage_path, storage_path)
usr_sudo = self.config['mycluster']['login']['image_user_sudo']
self.assertEqual(provider._sudo_user, usr_sudo)
sudo = self.config['mycluster']['login']['image_sudo']
self.assertEqual(provider._sudo, sudo)
def execute(self):
"""
Load the cluster and build a GC3Pie configuration snippet.
"""
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage,
include_config_dirs=True)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "mycluster", "hobbes", cloud_provider,
setup, nodes, configurator)
return cluster
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
# Get current cluster configuration
cluster_name = self.params.cluster
template = self.params.template
try:
cluster = configurator.load_cluster(cluster_name)
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def execute(self):
"""
Lists all nodes within the specified cluster with certain
information like id and ip.
"""
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
if self.params.update:
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def execute(self):
"""
Lists all nodes within the specified cluster with certain
information like id and ip.
"""
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage,
include_config_dirs=True)
cluster_name = self.params.cluster
try:
cluster = configurator.load_cluster(cluster_name)
if self.params.update:
cluster.update()
except (ClusterNotFound, ConfigurationError), ex:
log.error("Listing nodes from cluster %s: %s\n" %
(cluster_name, ex))
return
def test_create_cluster_storage(self):
# default storage path
configurator = Configurator(self.config)
storage = configurator.create_cluster_storage()
default_storage = configurator.general_conf['storage']
self.assertEqual(storage._storage_dir, default_storage)
# custom storage path
path = "/tmp"
configurator = Configurator(self.config, storage_path=path)
storage = configurator.create_cluster_storage()
self.assertEqual(storage._storage_dir, path)
def test_parsing_of_multiple_ansible_groups(self):
"""Fix regression causing multiple ansible groups to be incorrectly parsed
The bug caused this configuration snippet:
[setup/ansible]
frontend_groups=slurm_master,ganglia_frontend
to lead to the following inventory file:
[slurm_master,ganglia_frontend]
frontend001 ...
"""
cfg = minimal_configuration()
with open(self.cfgfile, 'w') as fd:
cfg.write(fd)
config = Configurator.fromConfig(self.cfgfile)
setup = config.create_setup_provider('c1')
self.assertEqual(setup.groups['misc'], ['misc_master', 'misc_client'])
def test_parsing_of_multiple_ansible_groups(self):
"""Fix regression causing multiple ansible groups to be incorrectly parsed
The bug caused this configuration snippet:
[setup/ansible]
frontend_groups=slurm_master,ganglia_frontend
to lead to the following inventory file:
[slurm_master,ganglia_frontend]
frontend001 ...
"""
cfg = minimal_configuration()
with open(self.cfgfile, 'w') as fd:
cfg.write(fd)
config = Configurator.fromConfig(self.cfgfile)
setup = config.create_setup_provider('c1')
self.assertEqual(setup.groups['misc'], ['misc_master', 'misc_client'])
def test_dump_cluster(self):
"""
Dump cluster to json
"""
storage = self.get_cluster_storage()
configurator = Configurator(Configuration().get_config(self.path))
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "cluster_name", "hobbes", MagicMock(),
MagicMock(), nodes, configurator)
instance_id = "test-id"
ip_public = "127.0.0.1"
ip_private = "127.0.0.2"
for node in cluster.get_all_nodes():
node.instance_id = instance_id
node.ip_public = ip_public
node.ip_private = ip_private
storage.dump_cluster(cluster)
dump = os.path.join(self.storage_path, "cluster_name.json")
f = open(dump, "r")
content = f.read()
expected = """
{"compute_nodes": 2, "nodes":
[{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute001",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute002",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "frontend", "name": "frontend001",
"ip_private": "127.0.0.2"}],
"frontend_nodes": 1, "name": "cluster_name",
"template": "mycluster"}"""
self.assertEqual(json.loads(content), json.loads(expected))
os.unlink(dump)
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
repository = configurator.create_repository()
clusters = repository.get_all()
if not clusters:
print("No clusters found.")
else:
print("""
The following clusters have been started.
Please note that there's no guarantee that they are fully configured:
""")
for cluster in sorted(clusters):
print("%s " % cluster.name)
print("-" * len(cluster.name))
print(" name: %s" % cluster.name)
for cls in cluster.nodes:
print(" - %s nodes: %d" % (cls, len(cluster.nodes[cls])))
print("")
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
repository = configurator.create_repository()
clusters = repository.get_all()
if not clusters:
print("No clusters found.")
else:
print("""
The following clusters have been started.
Please note that there's no guarantee that they are fully configured:
""")
for cluster in sorted(clusters):
print("%s " % cluster.name)
print("-" * len(cluster.name))
print(" name: %s" % cluster.name)
for cls in cluster.nodes:
print(" - %s nodes: %d" % (cls, len(cluster.nodes[cls])))
print("")
def execute(self):
configurator = Configurator.fromConfig(
self.params.config, storage_path=self.params.storage)
config = configurator.cluster_conf
print("""%d cluster templates found.""" % len(config))
templates = config.keys()
for pattern in self.params.clusters:
templates = [t for t in templates if fnmatch(t, pattern)]
for template in templates:
try:
cluster = configurator.create_cluster(template, template)
print("""
name: %s
cloud: %s""" % (template, cluster._cloud))
for nodetype in cluster.nodes:
print("%s nodes: %d" %
(nodetype, len(cluster.nodes[nodetype])))
except ConfigurationError, ex:
log.warning("unable to load cluster `%s`: %s", template, ex)
def _test_multiple_config_files(self, *paths_and_configs):
"""
Common code for all `test_read_multiple_config*` checks.
"""
tmpdir = None
try:
tmpdir = tempfile.mkdtemp()
paths = []
for path, content in paths_and_configs:
path = os.path.join(tmpdir, path)
basedir = os.path.dirname(path)
if not os.path.exists(basedir):
os.makedirs(basedir)
with open(path, 'w') as cfgfile:
cfgfile.write(content)
paths.append(path)
config = Configurator.fromConfig(paths)
# check all clusters are there
cfg = config.cluster_conf
self.assertTrue("slurm" in cfg)
# check for nodes
self.assertTrue("frontend" in cfg["slurm"]["nodes"])
self.assertTrue("compute" in cfg["slurm"]["nodes"])
# check one property in each category
self.assertEqual(cfg["slurm"]["cluster"]["security_group"],
"default")
self.assertEqual(cfg["slurm"]["login"]["image_user"], "ubuntu")
self.assertEqual(cfg["slurm"]["setup"]["provider"], "ansible")
self.assertEqual(cfg["slurm"]["cloud"]["ec2_region"], "nova")
finally:
if tmpdir:
shutil.rmtree(tmpdir)
def test_valid_minimal_configuration(self):
cfg = minimal_configuration(self.cfgfile)
with open(self.cfgfile, 'w') as fd:
cfg.write(fd)
config = Configurator.fromConfig(self.cfgfile)
def _check_read_config_object(self, cfgobj):
with open(self.cfgfile, 'wb') as fd:
cfgobj.write(fd)
ret = Configurator.fromConfig(self.cfgfile)
return ret
def _check_read_config(self, config):
with open(self.cfgfile, 'wb') as fd:
fd.write(config)
return Configurator.fromConfig(self.cfgfile)
def test_storage_type(self):
configurator = Configurator(self.config)
repo = configurator.create_repository()
def get_configurator(configfiles='~/.elasticluster/config', storage_path=None):
from elasticluster.conf import Configurator
conf = Configurator.fromConfig(configfiles, storage_path)
return conf