def create_cluster(self, template, name=None): """Creates a cluster by inspecting the configuration properties of the given cluster template. :param str template: name of the cluster template :param str name: name of the cluster. If not defined, the cluster will be named after the template. :return: :py:class:`elasticluster.cluster.cluster` instance: :raises ConfigurationError: cluster template not found in config """ if not name: name = template if template not in self.cluster_conf: raise ConfigurationError( "Invalid configuration for cluster `%s`: %s" "" % (template, name)) conf = self.cluster_conf[template] conf_login = self.cluster_conf[template]['login'] extra = conf['cluster'].copy() extra.pop('cloud') extra.pop('setup_provider') extra['template'] = template cluster = Cluster(name=name, cloud_provider=self.create_cloud_provider(template), setup_provider=self.create_setup_provider(template, name=name), user_key_name=conf_login['user_key_name'], user_key_public=conf_login['user_key_public'], user_key_private=conf_login["user_key_private"], repository=self.create_repository(), **extra) nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].items() if k.endswith('_nodes')) for kind, num in nodes.items(): conf_kind = conf['nodes'][kind] extra = conf_kind.copy() extra.pop('image_id', None) extra.pop('flavor', None) extra.pop('security_group', None) extra.pop('image_userdata', None) userdata = conf_kind.get('image_userdata', '') cluster.add_nodes(kind, num, conf_kind['image_id'], conf_login['image_user'], conf_kind['flavor'], conf_kind['security_group'], image_userdata=userdata, **extra) return cluster
def load(self, fp): data = yaml.safe_load(fp) if not data: raise ClusterError("Empty cluster state file: {0}".format(fp.name)) cluster = Cluster(**data) cluster.repository = self return cluster
def get_cluster(self, cloud_provider=None, config=None, nodes=None): if not cloud_provider: cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/", "nova", "a-key", "s-key") if not config: config = Configuration().get_config(self.path) setup = Mock() configurator = Configurator(config) conf_login = configurator.cluster_conf['mycluster']['login'] repository = PickleRepository(self.storage_path) cluster = Cluster( name="mycluster", cloud_provider=cloud_provider, setup_provider=setup, repository=repository, user_key_name=conf_login['user_key_name'], user_key_public=conf_login['user_key_public'], user_key_private=conf_login['user_key_private'], ) if not nodes: nodes = {"compute": 2, "frontend": 1} for kind, num in nodes.iteritems(): conf_kind = configurator.cluster_conf['mycluster']['nodes'][kind] cluster.add_nodes(kind, num, conf_kind['image_id'], conf_login['image_user'], conf_kind['flavor'], conf_kind['security_group']) return cluster
def create_cluster(self, template, name=None, cloud=None, setup=None): """ Creates a ``Cluster``:class: instance by inspecting the configuration properties of the given cluster template. :param str template: name of the cluster template :param str name: name of the cluster. If not defined, the cluster will be named after the template. :param cloud: A `CloudProvider`:py:class: instance to use instead of the configured one. If ``None`` (default) then the configured cloud provider will be used. :param setup: A `SetupProvider`:py:class: instance to use instead of the configured one. If ``None`` (default) then the configured setup provider will be used. :return: :py:class:`elasticluster.cluster.Cluster` instance: :raises ConfigurationError: cluster template not found in config """ if template not in self.cluster_conf: raise ConfigurationError( "No cluster template configuration by the name `{template}`" .format(template=template)) conf = self.cluster_conf[template] extra = conf.copy() extra.pop('cloud') extra.pop('nodes') extra.pop('setup') extra['template'] = template if cloud is None: cloud = self.create_cloud_provider(template) if name is None: name = template if setup is None: setup = self.create_setup_provider(template, name=name) cluster = Cluster( name=(name or template), cloud_provider=cloud, setup_provider=setup, user_key_name=conf['login']['user_key_name'], user_key_public=conf['login']['user_key_public'], user_key_private=conf['login']["user_key_private"], repository=self.create_repository(), **extra) nodes = conf['nodes'] for group_name in nodes: group_conf = nodes[group_name] for varname in ['image_user', 'image_userdata']: group_conf.setdefault(varname, conf['login'][varname]) cluster.add_nodes(group_name, **group_conf) return cluster
def get_cluster(self, cloud_provider=None, config=None, nodes=None): if not cloud_provider: cloud_provider = BotoCloudProvider("https://hobbes.gc3.uzh.ch/", "nova", "a-key", "s-key") if not config: config = Configuration().get_config(self.path) setup = Mock() configurator = Configurator(config) if not nodes: nodes = {"compute": 2, "frontend": 1} cluster = Cluster("mycluster", "mycluster", "hobbes", cloud_provider, setup, nodes, configurator) return cluster
def test_dump_cluster(self): """ Dump cluster to json """ storage = self.get_cluster_storage() configurator = Configurator(Configuration().get_config(self.path)) nodes = {"compute": 2, "frontend": 1} cluster = Cluster("mycluster", "cluster_name", "hobbes", MagicMock(), MagicMock(), nodes, configurator) instance_id = "test-id" ip_public = "127.0.0.1" ip_private = "127.0.0.2" for node in cluster.get_all_nodes(): node.instance_id = instance_id node.ip_public = ip_public node.ip_private = ip_private storage.dump_cluster(cluster) dump = os.path.join(self.storage_path, "cluster_name.json") f = open(dump, "r") content = f.read() expected = """ {"compute_nodes": 2, "nodes": [{"instance_id": "test-id", "ip_public": "127.0.0.1", "type": "compute", "name": "compute001", "ip_private": "127.0.0.2"}, {"instance_id": "test-id", "ip_public": "127.0.0.1", "type": "compute", "name": "compute002", "ip_private": "127.0.0.2"}, {"instance_id": "test-id", "ip_public": "127.0.0.1", "type": "frontend", "name": "frontend001", "ip_private": "127.0.0.2"}], "frontend_nodes": 1, "name": "cluster_name", "template": "mycluster"}""" self.assertEqual(json.loads(content), json.loads(expected)) os.unlink(dump)
def create_cluster(self, template, name=None): """ Creates a cluster by inspecting the configuration properties of the given cluster template. :param template: name of the cluster template :param name: name of the cluster. If not defined, the cluster will be named after the template. :return: :py:class:`elasticluster.cluster.cluster` instance :raises ConfigurationError: cluster template not found in config """ if not name: name = template if template not in self.cluster_conf: raise ConfigurationError( "Invalid configuration for cluster `%s`: %s" "" % (template, name)) conf = self.cluster_conf[template] nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].iteritems() if k.endswith('_nodes')) min_nodes = dict((k[:-10], int(v)) for k, v in conf['cluster'].iteritems() if k.endswith('_nodes_min')) extra = conf['cluster'].copy() extra.pop('cloud') extra.pop('setup_provider') return Cluster(template, name, conf['cluster']['cloud'], self.create_cloud_provider(template), self.create_setup_provider(template, name=name), nodes, self, min_nodes=min_nodes, **extra)
def load(self, fp): data = json.load(fp) cluster = Cluster(**data) cluster.repository = self return cluster