def create_cluster(self, template, name=None):
"""Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
conf_login = self.cluster_conf[template]['login']
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
extra['template'] = template
cluster = Cluster(name=name,
cloud_provider=self.create_cloud_provider(template),
setup_provider=self.create_setup_provider(template,
name=name),
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].items()
if k.endswith('_nodes'))
for kind, num in nodes.items():
conf_kind = conf['nodes'][kind]
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
userdata = conf_kind.get('image_userdata', '')
cluster.add_nodes(kind,
num,
conf_kind['image_id'],
conf_login['image_user'],
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
return cluster
def load(self, fp):
data = yaml.safe_load(fp)
if not data:
raise ClusterError("Empty cluster state file: {0}".format(fp.name))
cluster = Cluster(**data)
cluster.repository = self
return cluster
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
conf_login = configurator.cluster_conf['mycluster']['login']
repository = PickleRepository(self.storage_path)
cluster = Cluster(
name="mycluster",
cloud_provider=cloud_provider,
setup_provider=setup,
repository=repository,
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login['user_key_private'],
)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
for kind, num in nodes.iteritems():
conf_kind = configurator.cluster_conf['mycluster']['nodes'][kind]
cluster.add_nodes(kind, num, conf_kind['image_id'],
conf_login['image_user'], conf_kind['flavor'],
conf_kind['security_group'])
return cluster
def create_cluster(self, template, name=None, cloud=None, setup=None):
"""
Creates a ``Cluster``:class: instance by inspecting the configuration
properties of the given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:param cloud: A `CloudProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured cloud provider will be used.
:param setup: A `SetupProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured setup provider will be used.
:return: :py:class:`elasticluster.cluster.Cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if template not in self.cluster_conf:
raise ConfigurationError(
"No cluster template configuration by the name `{template}`"
.format(template=template))
conf = self.cluster_conf[template]
extra = conf.copy()
extra.pop('cloud')
extra.pop('nodes')
extra.pop('setup')
extra['template'] = template
if cloud is None:
cloud = self.create_cloud_provider(template)
if name is None:
name = template
if setup is None:
setup = self.create_setup_provider(template, name=name)
cluster = Cluster(
name=(name or template),
cloud_provider=cloud,
setup_provider=setup,
user_key_name=conf['login']['user_key_name'],
user_key_public=conf['login']['user_key_public'],
user_key_private=conf['login']["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = conf['nodes']
for group_name in nodes:
group_conf = nodes[group_name]
for varname in ['image_user', 'image_userdata']:
group_conf.setdefault(varname, conf['login'][varname])
cluster.add_nodes(group_name, **group_conf)
return cluster
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "mycluster", "hobbes", cloud_provider,
setup, nodes, configurator)
return cluster
def test_dump_cluster(self):
"""
Dump cluster to json
"""
storage = self.get_cluster_storage()
configurator = Configurator(Configuration().get_config(self.path))
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "cluster_name", "hobbes", MagicMock(),
MagicMock(), nodes, configurator)
instance_id = "test-id"
ip_public = "127.0.0.1"
ip_private = "127.0.0.2"
for node in cluster.get_all_nodes():
node.instance_id = instance_id
node.ip_public = ip_public
node.ip_private = ip_private
storage.dump_cluster(cluster)
dump = os.path.join(self.storage_path, "cluster_name.json")
f = open(dump, "r")
content = f.read()
expected = """
{"compute_nodes": 2, "nodes":
[{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute001",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute002",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "frontend", "name": "frontend001",
"ip_private": "127.0.0.2"}],
"frontend_nodes": 1, "name": "cluster_name",
"template": "mycluster"}"""
self.assertEqual(json.loads(content), json.loads(expected))
os.unlink(dump)
def create_cluster(self, template, name=None):
"""
Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param template: name of the cluster template
:param name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes'))
min_nodes = dict((k[:-10], int(v))
for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes_min'))
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
return Cluster(template,
name,
conf['cluster']['cloud'],
self.create_cloud_provider(template),
self.create_setup_provider(template, name=name),
nodes,
self,
min_nodes=min_nodes,
**extra)
def load(self, fp):
data = json.load(fp)
cluster = Cluster(**data)
cluster.repository = self
return cluster
