def _parse_flavor(self, flavor):
"""
Parse a flavor string into OpenNebula's `CPU=...` and `MEMORY=...` values.
"""
template_id = None
attributes = {}
parts = re.split(' *[,+\n] *', flavor, re.MULTILINE)
for part in parts:
if ':' not in part:
if template_id is None:
template_id = self._parse_template(part)
else:
raise ConfigurationError(
"Template ID or name given twice in flavor spec `{0}`".
format(flavor))
else:
key, value = part.split(':')
key = key.upper()
if key == 'TEMPLATE':
if template_id is None:
template_id = self._parse_template(part)
else:
raise ConfigurationError(
"Template ID or name given twice in flavor spec `{0}`"
.format(flavor))
else:
keys = key.split('.')
setitem_nested(attributes, keys, value)
# one of the two should have been filled by now
assert template_id or attributes
return template_id, attributes
def create_setup_provider(self, cluster_template, name=None):
"""Creates the setup provider for the given cluster template.
:param str cluster_template: template of the cluster
:param str name: name of the cluster to read configuration properties
"""
try:
conf_template = self.cluster_conf[cluster_template]
except KeyError as err:
raise ConfigurationError(
"No cluster template `{0}` found in configuration file".format(
cluster_template))
try:
conf = conf_template['setup']
except KeyError as err:
# this should have been caught during config validation!
raise ConfigurationError(
"No setup section for cluster template `{0}`"
" found in configuration file".format(cluster_template))
if name:
conf['cluster_name'] = name
conf_login = self.cluster_conf[cluster_template]['login']
provider_name = conf.get('provider', 'ansible')
if provider_name not in SETUP_PROVIDERS:
raise ConfigurationError(
"Invalid value `%s` for `setup_provider` in configuration "
"file." % provider_name)
provider = _get_provider(provider_name, SETUP_PROVIDERS)
storage_path = self.storage_path
playbook_path = conf.pop('playbook_path', None)
groups = self._read_node_groups(conf)
environment_vars = {}
for node_kind, grps in groups.iteritems():
if not isinstance(grps, list):
groups[node_kind] = [grps]
# Environment variables parsing
environment_vars[node_kind] = {}
for key, value in (
list(conf.items()) +
list(self.cluster_conf[cluster_template].items())):
# Set both group and global variables
for prefix in [(node_kind + '_var_'), "global_var_"]:
if key.startswith(prefix):
var = key.replace(prefix, '')
environment_vars[node_kind][var] = value
log.debug("setting variable %s=%s for node kind %s",
var, value, node_kind)
return provider(groups,
playbook_path=playbook_path,
environment_vars=environment_vars,
storage_path=storage_path,
sudo=conf_login['image_sudo'],
sudo_user=conf_login['image_user_sudo'],
**conf)
def __init__(self,
subscription_id,
tenant_id,
client_id,
secret,
location,
vm_deployment_template=None,
net_deployment_template=None,
**extra):
self.subscription_id = subscription_id
self.tenant_id = tenant_id
self.client_id = client_id
self.secret = secret
self.location = location
if vm_deployment_template:
try:
with open(vm_deployment_template) as template_file:
self.vm_deployment_template = json.load(template_file)
except Exception as err:
raise ConfigurationError(
"Could not load VM deployment template file `{0}`"
" in provider for Azure cloud: {1}".format(
vm_deployment_template, err))
else:
# Azure Resource Manager template for starting a new VM.
# Initially taken from:
# https://github.com/Azure-Samples/resource-manager-python-template-deployment/blob/master/templates/template.json
# Copyright (c) 2015 Microsoft Corporation
self.vm_deployment_template = json.loads(
resource_string('elasticluster',
'share/etc/azure_vm_template.json'))
if net_deployment_template:
try:
with open(net_deployment_template) as template_file:
self.net_deployment_template = json.load(template_file)
except Exception as err:
raise ConfigurationError(
"Could not load net deployment template file `{0}`"
" in provider for Azure cloud: {1}".format(
net_deployment_template, err))
else:
# Azure Resource Manager template for creating a new network.
self.net_deployment_template = json.loads(
resource_string('elasticluster',
'share/etc/azure_net_template.json'))
# these will be initialized later by `_init_az_api()`
self._compute_client = None
self._network_client = None
self._resource_client = None
# local state
self._inventory = {}
self._vm_details = {}
self._resource_groups_created = set()
self._networks_created = set()
def __init__(self, groups, playbook_path=None, environment_vars=None,
storage_path=None, sudo=True, sudo_user='******',
**extra_conf):
self.groups = groups
self._playbook_path = playbook_path
self.environment = environment_vars or {}
self._storage_path = storage_path
self._sudo_user = sudo_user
self._sudo = sudo
if 'ssh_pipelining' in extra_conf:
extra_conf['ansible_ssh_pipelining'] = extra_conf.pop('ssh_pipelining')
warn(
"Setup configuration option `ssh_pipelining`"
" has been renamed to `ansible_ssh_pipelining`."
" Please fix the configuration file(s), as support"
" for the old spelling will be removed in a future release.",
DeprecationWarning)
if 'ansible_module_dir' in extra_conf:
extra_conf['ansible_library'] = extra_conf.pop('ansible_module_dir')
warn(
"Setup configuration option `ansible_module_dir`"
" has been renamed to `ansible_library`."
" Please fix the configuration file(s), as support"
" for the old spelling will be removed in a future release.",
DeprecationWarning)
self.extra_conf = extra_conf
if not self._playbook_path:
# according to
# https://pythonhosted.org/setuptools/pkg_resources.html#resource-extraction
# requesting the filename to a directory causes all the
# contained files and directories to be extracted as well
playbook_dir = resource_filename('elasticluster', 'share/playbooks')
self._playbook_path = os.path.join(playbook_dir, 'site.yml')
else:
self._playbook_path = os.path.expanduser(self._playbook_path)
self._playbook_path = os.path.expandvars(self._playbook_path)
# sanity check
if not os.path.exists(self._playbook_path):
raise ConfigurationError(
"playbook `{playbook_path}` could not be found"
.format(playbook_path=self._playbook_path))
if not os.path.isfile(self._playbook_path):
raise ConfigurationError(
"playbook `{playbook_path}` is not a file"
.format(playbook_path=self._playbook_path))
if self._storage_path:
self._storage_path = os.path.expanduser(self._storage_path)
self._storage_path = os.path.expandvars(self._storage_path)
self._storage_path_tmp = False
if not os.path.exists(self._storage_path):
os.makedirs(self._storage_path)
else:
self._storage_path = tempfile.mkdtemp()
self._storage_path_tmp = True
def create_cloud_provider(self, cluster_template):
"""
Return cloud provider instance for the given cluster template.
:param str cluster_template: name of cluster template to use
:return: cloud provider instance that fulfills the contract of
:py:class:`elasticluster.providers.AbstractCloudProvider`
"""
cloud_conf = self.cluster_conf[cluster_template]['cloud']
provider = cloud_conf['provider']
print (CLOUD_PROVIDERS)
try:
ctor = _get_provider(provider, CLOUD_PROVIDERS)
except KeyError:
# this should have been caught during config validation!
raise ConfigurationError(
"Unknown cloud provider `{0}` for cluster `{1}`"
.format(provider, cluster_template))
except (ImportError, AttributeError) as err:
raise RuntimeError(
"Unable to load cloud provider `{0}`: {1}: {2}"
.format(provider, err.__class__.__name__, err))
provider_conf = cloud_conf.copy()
provider_conf.pop('provider')
# use a single keyword args dictionary for instanciating
# provider, so we can detect missing arguments in case of error
provider_conf['storage_path'] = self.storage_path
try:
return ctor(**provider_conf)
except TypeError:
# check that required parameters are given, and try to
# give a sensible error message if not; if we do not
# do this, users only see a message like this::
#
# ERROR Error: __init__() takes at least 5 arguments (4 given)
#
# which gives no clue about what to correct!
import inspect
args, varargs, keywords, defaults = inspect.getargspec(ctor.__init__)
if defaults is not None:
# `defaults` is a list of default values for the last N args
defaulted = dict((argname, value)
for argname, value in zip(reversed(args),
reversed(defaults)))
else:
# no default values at all
defaulted = {}
for argname in args[1:]: # skip `self`
if argname not in provider_conf and argname not in defaulted:
raise ConfigurationError(
"Missing required configuration parameter `{0}`"
" in cloud section for cluster `{1}`"
.format(argname, cluster_template))
def create_cluster(self, template, name=None):
"""Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
conf_login = self.cluster_conf[template]['login']
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
extra['template'] = template
cluster = Cluster(name=name,
cloud_provider=self.create_cloud_provider(template),
setup_provider=self.create_setup_provider(template,
name=name),
user_key_name=conf_login['user_key_name'],
user_key_public=conf_login['user_key_public'],
user_key_private=conf_login["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].items()
if k.endswith('_nodes'))
for kind, num in nodes.items():
conf_kind = conf['nodes'][kind]
extra = conf_kind.copy()
extra.pop('image_id', None)
extra.pop('flavor', None)
extra.pop('security_group', None)
extra.pop('image_userdata', None)
userdata = conf_kind.get('image_userdata', '')
cluster.add_nodes(kind,
num,
conf_kind['image_id'],
conf_login['image_user'],
conf_kind['flavor'],
conf_kind['security_group'],
image_userdata=userdata,
**extra)
return cluster
def pre_run(self):
self.params.extra_conf = {}
try:
if self.params.nodes:
nodes = self.params.nodes.split(',')
for nspec in nodes:
n, group = nspec.split(':')
if not n.isdigit():
raise ConfigurationError(
"Invalid syntax for option `--nodes`: "
"`%s` is not an integer." % n)
n = int(n)
self.params.extra_conf[group + '_nodes'] = n
except ValueError:
raise ConfigurationError(
"Invalid argument for option --nodes: %s" % self.params.nodes)
def create_cloud_provider(self, cluster_template):
"""
Return cloud provider instance for the given cluster template.
:param str cluster_template: name of cluster template to use
:return: cloud provider instance that fulfills the contract of
:py:class:`elasticluster.providers.AbstractCloudProvider`
"""
cloud_conf = self.cluster_conf[cluster_template]['cloud']
provider = cloud_conf['provider']
try:
ctor = _get_provider(provider, CLOUD_PROVIDERS)
except KeyError:
# this should have been caught during config validation!
raise ConfigurationError(
"Unknown cloud provider `{0}` for cluster `{1}`"
.format(provider, cluster_template))
except (ImportError, AttributeError) as err:
raise RuntimeError(
"Unable to load cloud provider `{0}`: {1}: {2}"
.format(provider, err.__class__.__name__, err))
provider_conf = cloud_conf.copy()
provider_conf.pop('provider')
return ctor(storage_path=self.storage_path, **provider_conf)
def execute(self):
"""
Starts a new cluster.
"""
cluster_template = self.params.cluster
if self.params.cluster_name:
cluster_name = self.params.cluster_name
else:
cluster_name = self.params.cluster
creator = make_creator(self.params.config,
storage_path=self.params.storage)
# overwrite configuration
cluster_nodes_conf = creator.cluster_conf[cluster_template]['nodes']
for kind, num in self.params.nodes_override.iteritems():
if kind not in cluster_nodes_conf:
raise ConfigurationError(
"No node group `{kind}` defined"
" in cluster template `{template}`"
.format(kind=kind, template=cluster_template))
cluster_nodes_conf[kind]['num'] = num
# First, check if the cluster is already created.
try:
cluster = creator.load_cluster(cluster_name)
except ClusterNotFound:
try:
cluster = creator.create_cluster(
cluster_template, cluster_name)
except ConfigurationError as err:
log.error("Starting cluster %s: %s", cluster_template, err)
return
try:
print("Starting cluster `{0}` with:".format(cluster.name))
for cls in cluster.nodes:
print("* {0:d} {1} nodes.".format(len(cluster.nodes[cls]), cls))
print("(This may take a while...)")
min_nodes = dict((kind, cluster_nodes_conf[kind]['min_num'])
for kind in cluster_nodes_conf)
cluster.start(min_nodes=min_nodes)
if self.params.no_setup:
print("NOT configuring the cluster as requested.")
else:
print("Configuring the cluster.")
print("(this too may take a while...)")
ret = cluster.setup()
if ret:
print("Your cluster is ready!")
else:
print("\nWARNING: YOUR CLUSTER IS NOT READY YET!")
print(cluster_summary(cluster))
except (KeyError, ImageError, SecurityGroupError, ClusterError) as err:
log.error("Could not start cluster `%s`: %s", cluster.name, err)
raise
def create_cluster(self, template, name=None, cloud=None, setup=None):
"""
Creates a ``Cluster``:class: instance by inspecting the configuration
properties of the given cluster template.
:param str template: name of the cluster template
:param str name: name of the cluster. If not defined, the cluster
will be named after the template.
:param cloud: A `CloudProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured cloud provider will be used.
:param setup: A `SetupProvider`:py:class: instance to use
instead of the configured one. If ``None`` (default)
then the configured setup provider will be used.
:return: :py:class:`elasticluster.cluster.Cluster` instance:
:raises ConfigurationError: cluster template not found in config
"""
if template not in self.cluster_conf:
raise ConfigurationError(
"No cluster template configuration by the name `{template}`"
.format(template=template))
conf = self.cluster_conf[template]
extra = conf.copy()
extra.pop('cloud')
extra.pop('nodes')
extra.pop('setup')
extra['template'] = template
if cloud is None:
cloud = self.create_cloud_provider(template)
if name is None:
name = template
if setup is None:
setup = self.create_setup_provider(template, name=name)
cluster = Cluster(
name=(name or template),
cloud_provider=cloud,
setup_provider=setup,
user_key_name=conf['login']['user_key_name'],
user_key_public=conf['login']['user_key_public'],
user_key_private=conf['login']["user_key_private"],
repository=self.create_repository(),
**extra)
nodes = conf['nodes']
for group_name in nodes:
group_conf = nodes[group_name]
for varname in ['image_user', 'image_userdata']:
group_conf.setdefault(varname, conf['login'][varname])
cluster.add_nodes(group_name, **group_conf)
return cluster
def _split_image_id(image_id):
try:
publisher, offer, sku, version = image_id.split('/', 3)
return (publisher, offer, sku, version)
except (ValueError, TypeError):
raise ConfigurationError(
"The 'image_id' parameter in Azure"
" has the form 'publisher/offer/sku/version'"
" (e.g., 'canonical/ubuntuserver/16.04.0-LTS/latest');"
" got '{0}' {1} instead!".format(image_id, type(image_id)))
def _read_one_auth():
"""
Read username and password from the ONE auth file,
or raise `ConfigurationError` if it cannot be found.
"""
one_auth_file = os.environ.get('ONE_AUTH',
os.path.expanduser('~/.one/one_auth'))
try:
with open(one_auth_file) as one_auth:
auth = one_auth.read().strip()
username, password = auth.split(':')
return username, password
except IOError:
raise ConfigurationError(
"Cannot read ONE auth file `{0}`.".format(one_auth_file))
except ValueError:
raise ConfigurationError(
"Cannot parse contents of file `{0}` as username:password pair."
.format(one_auth_file))
def create_setup_provider(self, cluster_template, name=None):
"""Creates the setup provider for the given cluster template.
:param str cluster_template: template of the cluster
:param str name: name of the cluster to read configuration properties
"""
conf = self.cluster_conf[cluster_template]['setup']
home = os.environ['HOME']
self.config = ConfigParser.ConfigParser()
self.path = home + "/.hwcc/config"
self.config.read(self.path)
sfs_items = self.config.items("sfs")
if name:
conf['cluster_name'] = name
conf_login = self.cluster_conf[cluster_template]['login']
provider_name = conf.get('provider', 'ansible')
if provider_name not in SETUP_PROVIDERS:
raise ConfigurationError(
"Invalid value `%s` for `setup_provider` in configuration "
"file." % provider_name)
provider = _get_provider(provider_name, SETUP_PROVIDERS)
storage_path = self.storage_path
playbook_path = conf.pop('playbook_path', None)
groups = self._read_node_groups(conf)
environment_vars = {}
for node_kind, grps in groups.iteritems():
if not isinstance(grps, list):
groups[node_kind] = [grps]
# Environment variables parsing
environment_vars[node_kind] = {}
for key, value in (
list(conf.items()) +
list(self.cluster_conf[cluster_template].items()) +
list(sfs_items)):
# Set both group and global variables
for prefix in [(node_kind + '_var_'), "global_var_", "sfs_",
"master_"]:
if key.startswith(prefix):
var = key.replace(prefix, '')
environment_vars[node_kind][var] = value
log.debug("setting variable %s=%s for node kind %s",
var, value, node_kind)
return provider(groups,
playbook_path=playbook_path,
environment_vars=environment_vars,
storage_path=storage_path,
sudo=conf_login['image_sudo'],
sudo_user=conf_login['image_user_sudo'],
**conf)
def pre_run(self):
self.params.nodes_to_add = {}
self.params.nodes_to_remove = {}
try:
if self.params.add:
nodes = self.params.add.split(',')
for nspec in nodes:
n, group = nspec.split(':')
if not n.isdigit():
raise ConfigurationError(
"Invalid syntax for option `--nodes`: "
"`%s` is not an integer." % n)
self.params.nodes_to_add[group] = int(n)
if self.params.remove:
nodes = self.params.remove.split(',')
for nspec in nodes:
n, group = nspec.split(':')
self.params.nodes_to_remove[group] = int(n)
except ValueError as ex:
raise ConfigurationError("Invalid syntax for argument: %s" % ex)
def create_setup_provider(self, cluster_template, name=None):
"""Creates the setup provider for the given cluster template.
:param str cluster_template: template of the cluster
:param str name: name of the cluster to read configuration properties
"""
conf = self.cluster_conf[cluster_template]['setup']
conf['general_conf'] = self.general_conf.copy()
if name:
conf['cluster_name'] = name
conf_login = self.cluster_conf[cluster_template]['login']
provider_name = conf.get('provider')
if provider_name not in Configurator.setup_providers_map:
raise ConfigurationError(
"Invalid value `%s` for `setup_provider` in configuration "
"file." % provider_name)
storage_path = self.general_conf['storage_path']
if 'playbook_path' in conf:
playbook_path = conf['playbook_path']
del conf['playbook_path']
else:
playbook_path = None
groups = dict((k[:-7], v.split(',')) for k, v in conf.items()
if k.endswith('_groups'))
environment = dict()
for nodekind, grps in groups.items():
if not isinstance(grps, list):
groups[nodekind] = [grps]
# Environment variables parsing
environment[nodekind] = dict()
for key, value in list(conf.items()) + list(
self.cluster_conf[cluster_template]['cluster'].items()):
# Set both group and global variables
for prefix in ["%s_var_" % nodekind, "global_var_"]:
if key.startswith(prefix):
var = key.replace(prefix, '')
environment[nodekind][var] = value
log.debug("setting variable %s=%s for node kind %s",
var, value, nodekind)
provider = Configurator.setup_providers_map[provider_name]
return provider(groups,
playbook_path=playbook_path,
environment_vars=environment,
storage_path=storage_path,
sudo_user=conf_login['image_user_sudo'],
sudo=conf_login['image_sudo'],
**conf)
def pre_run(self):
self.params.nodes_override = {}
if self.params.nodes:
nodes = self.params.nodes.split(',')
for nspec in nodes:
n, kind = nspec.split(':')
try:
n = int(n)
except (ValueError, TypeError) as err:
raise ConfigurationError(
"Invalid syntax for option `--nodes`: "
"cannot convert `{n}` to integer: {err}".format(
n=n, err=err))
self.params.nodes_override[kind] = n
def _dereference_config_tree(tree, evict_on_error=True):
# FIXME: Should allow *three* distinct behaviors on error?
# - "evict on error": remove the offending section and continue
# - "raise exception": raise a ConfigurationError at the first error
# - "just report": log errors but try to return all that makes sense
"""
Modify `tree` in-place replacing cross-references by section name with the
actual section content.
For example, if a cluster section lists a key/value pair
``'login': '******'``, this will be replaced with ``'login': { ... }``.
"""
to_evict = []
for cluster_name, cluster_conf in tree['cluster'].iteritems():
for key in ['cloud', 'login', 'setup']:
try:
refname = cluster_conf[key]
except KeyError:
log.error(
"Configuration section `cluster/%s`"
" is missing a `%s=` section reference."
" %s",
cluster_name, key,
("Dropping cluster definition." if evict_on_error else ""))
if evict_on_error:
to_evict.append(cluster_name)
break
else:
# cannot continue
raise ConfigurationError(
"Invalid cluster definition `cluster/{0}:"
" missing `{1}=` configuration key"
.format(cluster_name, key))
try:
# dereference
cluster_conf[key] = tree[key][refname]
except KeyError:
log.error(
"Configuration section `cluster/%s`"
" references non-existing %s section `%s`."
" %s",
cluster_name, key, refname,
("Dropping cluster definition." if evict_on_error else ""))
if evict_on_error:
to_evict.append(cluster_name)
break
for cluster_name in to_evict:
del tree['cluster'][cluster_name]
return tree
def _find_template_by_name(self, name):
"""
Return ID of template whose name is exactly *name*.
"""
with self._api_lock:
templates = self.server.templatepool.info(
-1, # Connected user's and his group's resources
-1, # range start, use -1 for "no restriction"
-1, # range end, use -1 for "no restriction"
)
# FIXME: I'm unsure whether accessing attributes of
# `template` can trigger a transparent XML-RPC call... for
# safety, run this loop while holding the API lock.
for template in templates.VMTEMPLATE:
if template.NAME == name:
return template.ID
raise ConfigurationError(
"No VM template found by the name `{0}`".format(name))
def create_setup_provider(self, cluster_template, name=None):
conf = self.cluster_conf[cluster_template]['setup']
conf['general_conf'] = self.general_conf.copy()
if name:
conf['cluster_name'] = name
conf_login = self.cluster_conf[cluster_template]['login']
provider_name = conf.get('provider')
if provider_name not in Configurator.setup_providers_map:
raise ConfigurationError(
"Invalid value `%s` for `setup_provider` in configuration "
"file." % provider_name)
provider = Configurator.setup_providers_map[provider_name]
return provider(conf_login['user_key_private'],
conf_login['image_user'],
conf_login['image_user_sudo'],
conf_login['image_sudo'], **conf)
def create_cluster(self, template, name=None):
"""
Creates a cluster by inspecting the configuration properties of the
given cluster template.
:param template: name of the cluster template
:param name: name of the cluster. If not defined, the cluster
will be named after the template.
:return: :py:class:`elasticluster.cluster.cluster` instance
:raises ConfigurationError: cluster template not found in config
"""
if not name:
name = template
if template not in self.cluster_conf:
raise ConfigurationError(
"Invalid configuration for cluster `%s`: %s"
"" % (template, name))
conf = self.cluster_conf[template]
nodes = dict((k[:-6], int(v)) for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes'))
min_nodes = dict((k[:-10], int(v))
for k, v in conf['cluster'].iteritems()
if k.endswith('_nodes_min'))
extra = conf['cluster'].copy()
extra.pop('cloud')
extra.pop('setup_provider')
return Cluster(template,
name,
conf['cluster']['cloud'],
self.create_cloud_provider(template),
self.create_setup_provider(template, name=name),
nodes,
self,
min_nodes=min_nodes,
**extra)
def get_ssh_to_node(self, ssh_to=None):
"""
Return target node for SSH/SFTP connections.
The target node is the first node of the class specified in
the configuration file as ``ssh_to`` (but argument ``ssh_to``
can override this choice).
If not ``ssh_to`` has been specified in this cluster's config,
then try node class names ``ssh``, ``login``, ``frontend``,
and ``master``: if any of these is non-empty, return the first
node.
If all else fails, return the first node of the first class
(in alphabetic order).
:return: :py:class:`Node`
:raise: :py:class:`elasticluster.exceptions.NodeNotFound`
if no valid frontend node is found
"""
if ssh_to is None:
ssh_to = self.ssh_to
# first try to interpret `ssh_to` as a node name
if ssh_to:
try:
return self.get_node_by_name(ssh_to)
except NodeNotFound:
pass
# next, ensure `ssh_to` is a class name
if ssh_to:
try:
parts = self._naming_policy.parse(ssh_to)
log.warning(
"Node `%s` not found."
" Trying to find other node in class `%s` ...", ssh_to,
parts['kind'])
ssh_to = parts['kind']
except ValueError:
# it's already a class name
pass
# try getting first node of kind `ssh_to`
if ssh_to:
try:
nodes = self.nodes[ssh_to]
except KeyError:
raise ConfigurationError(
"Invalid configuration item `ssh_to={ssh_to}` in cluster `{name}`:"
" node class `{ssh_to}` does not exist in this cluster.".
format(ssh_to=ssh_to, name=self.name))
try:
return nodes[0]
except IndexError:
log.warning(
"Chosen `ssh_to` class `%s` is empty: unable to "
"get the choosen frontend node from that class.", ssh_to)
# If we reach this point, `ssh_to` was not set or the
# preferred class was empty. Try "natural" `ssh_to` values.
for kind in ['ssh', 'login', 'frontend', 'master']:
try:
nodes = self.nodes[kind]
return nodes[0]
except (KeyError, IndexError):
pass
# ... if all else fails, return first node
for kind in sorted(self.nodes.keys()):
if self.nodes[kind]:
return self.nodes[kind][0]
# Uh-oh, no nodes in this cluster!
raise NodeNotFound("Unable to find a valid frontend:"
" cluster has no nodes!")
def start_instance(self,
key_name,
public_key_path,
private_key_path,
security_group,
flavor,
image_id,
image_userdata,
username=None,
node_name=None,
**kwargs):
"""Starts a new instance on the cloud using the given properties.
The following tasks are done to start an instance:
* establish a connection to the cloud web service
* check ssh keypair and upload it if it does not yet exist. This is
a locked process, since this function might be called in multiple
threads and we only want the key to be stored once.
* check if the security group exists
* run the instance with the given properties
:param str key_name: name of the ssh key to connect
:param str public_key_path: path to ssh public key
:param str private_key_path: path to ssh private key
:param str security_group: firewall rule definition to apply on the
instance
:param str flavor: machine type to use for the instance
:param str image_id: image type (os) to use for the instance
:param str image_userdata: command to execute after startup
:param str username: username for the given ssh key, default None
:return: str - instance id of the started instance
"""
self._init_os_api()
vm_start_args = {}
log.debug("Checking keypair `%s` ...", key_name)
with OpenStackCloudProvider.__node_start_lock:
self._check_keypair(key_name, public_key_path, private_key_path)
vm_start_args['key_name'] = key_name
security_groups = [sg.strip() for sg in security_group.split(',')]
self._check_security_groups(security_groups)
vm_start_args['security_groups'] = security_groups
# Check if the image id is present.
if image_id not in [img.id for img in self._get_images()]:
raise ImageError(
"No image found with ID `{0}` in project `{1}` of cloud {2}".
format(image_id, self._os_tenant_name, self._os_auth_url))
vm_start_args['userdata'] = image_userdata
# Check if the flavor exists
flavors = [fl for fl in self._get_flavors() if fl.name == flavor]
if not flavors:
raise FlavorError(
"No flavor found with name `{0}` in project `{1}` of cloud {2}"
.format(flavor, self._os_tenant_name, self._os_auth_url))
flavor = flavors[0]
network_ids = [
net_id.strip()
for net_id in kwargs.pop('network_ids', '').split(',')
]
if network_ids:
nics = [{
'net-id': net_id,
'v4-fixed-ip': ''
} for net_id in network_ids]
log.debug("Specifying networks for node %s: %s", node_name,
', '.join([nic['net-id'] for nic in nics]))
else:
nics = None
vm_start_args['nics'] = nics
if 'boot_disk_size' in kwargs:
# check if the backing volume is already there
volume_name = '{name}-{id}'.format(name=node_name, id=image_id)
if volume_name in [v.name for v in self._get_volumes()]:
raise ImageError(
"Volume `{0}` already exists in project `{1}` of cloud {2}"
.format(volume_name, self._os_tenant_name,
self._os_auth_url))
log.info('Creating volume `%s` to use as VM disk ...', volume_name)
try:
bds = int(kwargs['boot_disk_size'])
if bds < 1:
raise ValueError('non-positive int')
except (ValueError, TypeError):
raise ConfigurationError(
"Invalid `boot_disk_size` specified:"
" should be a positive integer, got {0} instead".format(
kwargs['boot_disk_size']))
volume = self.cinder_client.volumes.create(
size=bds,
name=volume_name,
imageRef=image_id,
volume_type=kwargs.pop('boot_disk_type'))
# wait for volume to come up
volume_available = False
while not volume_available:
for v in self._get_volumes():
if v.name == volume_name and v.status == 'available':
volume_available = True
break
sleep(1) # FIXME: hard-coded waiting time
# ok, use volume as VM disk
vm_start_args['block_device_mapping'] = {
# FIXME: is it possible that `vda` is not the boot disk? e.g. if
# a non-paravirtualized kernel is being used? should we allow
# to set the boot device as an image parameter?
'vda':
('{id}:::{delete_on_terminate}'.format(id=volume.id,
delete_on_terminate=1)),
}
# due to some `nova_client.servers.create()` implementation weirdness,
# the first three args need to be spelt out explicitly and cannot be
# conflated into `**vm_start_args`
vm = self.nova_client.servers.create(node_name, image_id, flavor,
**vm_start_args)
# allocate and attach a floating IP, if requested
if self.request_floating_ip:
# We need to list the floating IPs for this instance
try:
# python-novaclient <8.0.0
floating_ips = [
ip for ip in self.nova_client.floating_ips.list()
if ip.instance_id == vm.id
]
except AttributeError:
floating_ips = self.neutron_client.list_floatingips(id=vm.id)
# allocate new floating IP if none given
if not floating_ips:
self._allocate_address(vm, network_ids)
self._instances[vm.id] = vm
return vm.id
def start_instance(self,
key_name,
public_key_path,
private_key_path,
security_group,
flavor,
image_id,
image_userdata,
username='******',
node_name=None,
boot_disk_size=30,
storage_account_type='Standard_LRS',
**extra):
"""
Start a new VM using the given properties.
:param str key_name:
**unused in Azure**, only present for interface compatibility
:param str public_key_path:
path to ssh public key to authorize on the VM (for user `username`, see below)
:param str private_key_path:
**unused in Azure**, only present for interface compatibility
:param str security_group:
network security group to attach VM to, **currently unused**
:param str flavor:
machine type to use for the instance
:param str image_id:
disk image to use for the instance;
has the form *publisher/offer/sku/version*
(e.g., ``canonical/ubuntuserver/16.04.0-LTS/latest``)
:param str image_userdata:
command to execute after startup, **currently unused**
:param int boot_disk_size:
size of boot disk to use; values are specified in gigabytes.
:param str username:
username for the given ssh key
(default is ``root`` as it's always guaranteed to exist,
but you probably don't want to use that)
:param str storage_account_type:
Type of disks to attach to the VM. For a list of valid values,
see: https://docs.microsoft.com/en-us/rest/api/compute/disks/createorupdate#diskstorageaccounttypes
:return: tuple[str, str] -- resource group and node name of the started VM
"""
self._init_az_api()
# Warn of unsupported parameters, if set. We do not warn
# about `user_key` or `private_key_path` since they come from
# a `[login/*]` section and those can be shared across
# different cloud providers.
if security_group and security_group != 'default':
warn("Setting `security_group` is currently not supported"
" in the Azure cloud; VMs will all be attached to"
" a network security group named after the cluster name.")
if image_userdata:
warn("Parameter `image_userdata` is currently not supported"
" in the Azure cloud and will be ignored.")
# Use the cluster name to identify the Azure resource group;
# however, `Node.cluster_name` is not passed down here so
# extract it from the node name, which always contains it as
# the substring before the leftmost dash (see `cluster.py`,
# line 1182)
cluster_name, _ = node_name.rsplit('-', 1)
if not self.__compiled_pattern_for_names.match(cluster_name):
raise ConfigurationError(
"The cluster name `{0}` does not match the Azure requirement for names. "
"Only numbers, lowercase letters and dashes are allowed, "
"the value must begin with a lowercase letter and cannot end with a slash, "
"and must also be less than 63 characters long.".format(
cluster_name))
if not self.__compiled_pattern_for_names.match(node_name):
raise ConfigurationError(
"The node name `{0}` does not match the Azure requirement for names. "
"Only numbers, lowercase letters and dashes are allowed, "
"the value must begin with a lowercase letter and cannot end with a slash, "
"and must also be less than 63 characters long.".format(
node_name))
with self.__lock:
if cluster_name not in self._resource_groups_created:
self._resource_client.resource_groups.create_or_update(
cluster_name, {'location': self.location})
self._resource_groups_created.add(cluster_name)
# read public SSH key
with open(public_key_path, 'r') as public_key_file:
public_key = public_key_file.read()
image_publisher, image_offer, \
image_sku, image_version = self._split_image_id(image_id)
if not security_group:
security_group = (cluster_name + '-secgroup')
net_parameters = {
'networkSecurityGroupName': {
'value': security_group,
},
'subnetName': {
'value': cluster_name
},
}
net_name = net_parameters['subnetName']['value']
with self.__lock:
if net_name not in self._networks_created:
log.debug("Creating network `%s` in Azure ...", net_name)
oper = self._resource_client.deployments.create_or_update(
cluster_name, net_name, {
'mode': DeploymentMode.incremental,
'template': self.net_deployment_template,
'parameters': net_parameters,
})
oper.wait()
self._networks_created.add(net_name)
boot_disk_size_gb = int(boot_disk_size)
vm_parameters = {
'adminUserName': {
'value': username
},
'imagePublisher': {
'value': image_publisher
}, # e.g., 'canonical'
'imageOffer': {
'value': image_offer
}, # e.g., ubuntuserver
'imageSku': {
'value': image_sku
}, # e.g., '16.04.0-LTS'
'imageVersion': {
'value': image_version
}, # e.g., 'latest'
'networkSecurityGroupName': {
'value': security_group,
},
'sshKeyData': {
'value': public_key
},
'storageAccountName': {
'value':
self._make_storage_account_name(cluster_name, node_name)
},
'storageAccountType': {
'value': storage_account_type
},
'subnetName': {
'value': cluster_name
},
'vmName': {
'value': node_name
},
'vmSize': {
'value': flavor
},
'bootDiskSize': {
'value': boot_disk_size_gb
}
}
log.debug("Deploying `%s` VM template to Azure ...",
vm_parameters['vmName']['value'])
oper = self._resource_client.deployments.create_or_update(
cluster_name, node_name, {
'mode': DeploymentMode.incremental,
'template': self.vm_deployment_template,
'parameters': vm_parameters,
})
oper.wait()
# the `instance_id` is a composite type since we need both the
# resource group name and the vm name to uniquely identify a VM
return [cluster_name, node_name]