def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
conf_login = configurator.cluster_conf['mycluster']['login']
repository = PickleRepository(self.storage_path)
cluster = Cluster(
name="mycluster",
cloud_provider=cloud_provider,
setup_provider=setup,
repository=repository,
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login['user_key_private'],
)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
for kind, num in nodes.iteritems():
conf_kind = configurator.cluster_conf['mycluster']['nodes'][kind]
cluster.add_nodes(kind, num, conf_kind['image_id'],
conf_login['image_user'], conf_kind['flavor'],
conf_kind['security_group'])
return cluster
def load(self, fp):
data = yaml.safe_load(fp)
if not data:
raise ClusterError("Empty cluster state file: {0}".format(fp.name))
cluster = Cluster(**data)
cluster.repository = self
return cluster
def create_cluster(self, template, name=None):
"""Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
conf_login = self.cluster_conf[template]['login']
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
extra['template'] = template
cluster = Cluster(name=name,
cloud_provider=self.create_cloud_provider(template),
setup_provider=self.create_setup_provider(template,
name=name),
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].items()
if k.endswith('_nodes'))
for kind, num in nodes.items():
conf_kind = conf['nodes'][kind]
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
userdata = conf_kind.get('image_userdata', '')
cluster.add_nodes(kind,
num,
conf_kind['image_id'],
conf_login['image_user'],
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
return cluster
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
conf_login = configurator.cluster_conf['mycluster']['login']
repository = PickleRepository(self.storage_path)
cluster = Cluster(name="mycluster",
cloud_provider=cloud_provider,
setup_provider=setup,
repository=repository,
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login['user_key_private'],
)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
for kind, num in nodes.iteritems():
conf_kind = configurator.cluster_conf['mycluster']['nodes'][kind]
cluster.add_nodes(kind, num, conf_kind['image_id'],
conf_login['image_user'],
conf_kind['flavor'],
conf_kind['security_group'])
return cluster
def load(self, fp):
data = yaml.safe_load(fp)
if not data:
raise ClusterError("Empty cluster state file: {0}".format(fp.name))
cluster = Cluster(**data)
cluster.repository = self
return cluster
def create_cluster(self, template, name=None):
"""Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
conf_login = self.cluster_conf[template]['login']
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
extra['template'] = template
cluster = Cluster(name=name,
cloud_provider=self.create_cloud_provider(template),
setup_provider=self.create_setup_provider(template, name=name),
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = dict(
(k[:-6], int(v)) for k, v in conf['cluster'].iteritems() if
k.endswith('_nodes'))
for kind, num in nodes.iteritems():
conf_kind = conf['nodes'][kind]
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
userdata = conf_kind.get('image_userdata', '')
cluster.add_nodes(kind,
num,
conf_kind['image_id'],
conf_login['image_user'],
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
return cluster
def create_cluster(self, template, name=None, cloud=None, setup=None):
"""
Creates a ``Cluster``:class: instance by inspecting the configuration
properties of the given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:param cloud: A `CloudProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured cloud provider will be used.
:param setup: A `SetupProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured setup provider will be used.
:return: :py:class:`elasticluster.cluster.Cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if template not in self.cluster_conf:
raise ConfigurationError(
"No cluster template configuration by the name `{template}`"
.format(template=template))
conf = self.cluster_conf[template]
extra = conf.copy()
extra.pop('cloud')
extra.pop('nodes')
extra.pop('setup')
extra['template'] = template
if cloud is None:
cloud = self.create_cloud_provider(template)
if name is None:
name = template
if setup is None:
setup = self.create_setup_provider(template, name=name)
cluster = Cluster(
name=(name or template),
cloud_provider=cloud,
setup_provider=setup,
user_key_name=conf['login']['user_key_name'],
user_key_public=conf['login']['user_key_public'],
user_key_private=conf['login']["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = conf['nodes']
for group_name in nodes:
group_conf = nodes[group_name]
for varname in ['image_user', 'image_userdata']:
group_conf.setdefault(varname, conf['login'][varname])
cluster.add_nodes(group_name, **group_conf)
return cluster
def create_cluster(self, template, name=None, cloud=None, setup=None):
"""
Creates a ``Cluster``:class: instance by inspecting the configuration
properties of the given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:param cloud: A `CloudProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured cloud provider will be used.
:param setup: A `SetupProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured setup provider will be used.
:return: :py:class:`elasticluster.cluster.Cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if template not in self.cluster_conf:
raise ConfigurationError(
"No cluster template configuration by the name `{template}`"
.format(template=template))
conf = self.cluster_conf[template]
extra = conf.copy()
extra.pop('cloud')
extra.pop('nodes')
extra.pop('setup')
extra['template'] = template
if cloud is None:
cloud = self.create_cloud_provider(template)
if name is None:
name = template
if setup is None:
setup = self.create_setup_provider(template, name=name)
cluster = Cluster(
name=(name or template),
cloud_provider=cloud,
setup_provider=setup,
user_key_name=conf['login']['user_key_name'],
user_key_public=conf['login']['user_key_public'],
user_key_private=conf['login']["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = conf['nodes']
for group_name in nodes:
group_conf = nodes[group_name]
for varname in ['image_user', 'image_userdata']:
group_conf.setdefault(varname, conf['login'][varname])
cluster.add_nodes(group_name, **group_conf)
return cluster
def setup(self):
cloud_provider = MagicMock()
setup_provider = MagicMock()
setup_provider.setup_cluster.return_value = True
cluster = Cluster(config_cluster_name, config_cloud_name, cloud_provider, setup_provider, 1, 2, Configurator())
cluster.setup()
setup_provider.setup_cluster.assert_called_once_with(cluster)
def test_load_from_storage(self):
cloud_provider = MagicMock()
cloud_provider.is_instance_running.return_value = True
cloud_provider.get_ips.return_value = ('127.0.0.1', '127.0.0.1')
setup_provider = MagicMock()
storage = MagicMock()
cluster = Cluster(config_cluster_name, config_cloud_name, cloud_provider, setup_provider, 1, 2, Configurator())
cluster._storage = storage
cluster.load_from_storage()
storage.load_cluster.assert_called_once_with(cluster)
def test_add_node(self):
cloud_provider = BotoCloudProvider(config_cloud_ec2_url, config_cloud_ec2_region, config_cloud_ec2_access_key, config_cloud_ec2_secret_key)
setup_provider = AnsibleSetupProvider(config_login_user_key_private, config_login_image_user, config_login_image_user_sudo, config_login_image_sudo, config_setup_playbook_path, config_setup_frontend_groups, config_setup_compute_groups)
cluster = Cluster(config_cluster_name, config_cloud_name, cloud_provider, setup_provider, 1, 2, Configurator())
frontend_amount = len(cluster.frontend_nodes)
frontend_node = cluster.add_node(Node.frontend_type)
assert frontend_amount == (len(cluster.frontend_nodes) - 1)
assert frontend_node == cluster.frontend_nodes[-1]
assert frontend_node.type == Node.frontend_type
compute_amount = len(cluster.compute_nodes)
compute_node = cluster.add_node(Node.compute_type)
assert compute_amount == (len(cluster.compute_nodes) - 1)
assert compute_node == cluster.compute_nodes[-1]
assert compute_node.type == Node.compute_type
def test_dump_cluster(self):
"""
Dump cluster to json
"""
storage = self.get_cluster_storage()
configurator = Configurator(Configuration().get_config(self.path))
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "cluster_name", "hobbes", MagicMock(),
MagicMock(), nodes, configurator)
instance_id = "test-id"
ip_public = "127.0.0.1"
ip_private = "127.0.0.2"
for node in cluster.get_all_nodes():
node.instance_id = instance_id
node.ip_public = ip_public
node.ip_private = ip_private
storage.dump_cluster(cluster)
dump = os.path.join(self.storage_path, "cluster_name.json")
f = open(dump, "r")
content = f.read()
expected = """
{"compute_nodes": 2, "nodes":
[{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute001",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "compute", "name": "compute002",
"ip_private": "127.0.0.2"},
{"instance_id": "test-id", "ip_public": "127.0.0.1",
"type": "frontend", "name": "frontend001",
"ip_private": "127.0.0.2"}],
"frontend_nodes": 1, "name": "cluster_name",
"template": "mycluster"}"""
self.assertEqual(json.loads(content), json.loads(expected))
os.unlink(dump)
def test_start(self):
cloud_provider = MagicMock()
cloud_provider.start_instance.return_value = u'test-id'
cloud_provider.get_ips.return_value = ('127.0.0.1', '127.0.0.1')
running_states = [True, True, True, True, True, False, False, False, False, False]
def side_effect_is_instance_running(id):
return running_states.pop()
cloud_provider.is_instance_running.side_effect = side_effect_is_instance_running
setup_provider = MagicMock()
setup_provider.setup_cluster.return_value = True
cluster = Cluster(config_cluster_name, config_cloud_name, cloud_provider, setup_provider, 1, 2, Configurator())
cluster._storage = MagicMock()
cluster.start()
cluster._storage.dump_cluster.assert_called_once_with(cluster)
for node in cluster.frontend_nodes + cluster.compute_nodes:
assert node.instance_id == u'test-id'
assert node.ip_public == '127.0.0.1'
assert node.ip_private == '127.0.0.1'
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/",
"nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
cluster = Cluster("mycluster", "mycluster", "hobbes", cloud_provider,
setup, nodes, configurator)
return cluster
def test_remove_node(self):
cloud_provider = BotoCloudProvider(config_cloud_ec2_url, config_cloud_ec2_region, config_cloud_ec2_access_key, config_cloud_ec2_secret_key)
setup_provider = AnsibleSetupProvider(config_login_user_key_private, config_login_image_user, config_login_image_user_sudo, config_login_image_sudo, config_setup_playbook_path, config_setup_frontend_groups, config_setup_compute_groups)
cluster = Cluster(config_cluster_name, config_cloud_name, cloud_provider, setup_provider, 1, 2, Configurator())
frontend_amount = len(cluster.frontend_nodes)
frontend_node = cluster.frontend_nodes[-1]
frontend_node_name = frontend_node.name
cluster.remove_node(frontend_node)
assert frontend_amount == (len(cluster.frontend_nodes) + 1)
for node in cluster.frontend_nodes:
assert node.name != frontend_node_name
compute_amount = len(cluster.compute_nodes)
compute_node = cluster.compute_nodes[-1]
compute_node_name = compute_node.name
cluster.remove_node(compute_node)
assert compute_amount == (len(cluster.compute_nodes) + 1)
for node in cluster.compute_nodes:
assert node.name != compute_node_name
def get_cluster(self, cloud_provider=None, config=None, nodes=None):
if not cloud_provider:
cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/", "nova", "a-key", "s-key")
if not config:
config = Configuration().get_config(self.path)
setup = Mock()
configurator = Configurator(config)
conf_login = configurator.cluster_conf["mycluster"]["login"]
repository = PickleRepository(self.storage_path)
cluster = Cluster(
"mycluster",
cloud_provider,
setup,
repository,
conf_login["user_key_name"],
conf_login["user_key_public"],
conf_login["user_key_private"],
)
if not nodes:
nodes = {"compute": 2, "frontend": 1}
for kind, num in nodes.iteritems():
conf_kind = configurator.cluster_conf["mycluster"]["nodes"][kind]
cluster.add_nodes(
kind,
num,
conf_kind["image_id"],
conf_login["image_user"],
conf_kind["flavor"],
conf_kind["security_group"],
)
return cluster
def create_cluster(self, template, name=None):
"""
Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param template: name of the cluster template
:param name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes'))
min_nodes = dict((k[:-10], int(v))
for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes_min'))
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
return Cluster(template,
name,
conf['cluster']['cloud'],
self.create_cloud_provider(template),
self.create_setup_provider(template, name=name),
nodes,
self,
min_nodes=min_nodes,
**extra)
def load(self, fp):
data = json.load(fp)
cluster = Cluster(**data)
cluster.repository = self
return cluster
def load(self, fp):
data = json.load(fp)
cluster = Cluster(**data)
cluster.repository = self
return cluster
