def run(self):
while not self.stoprequest.isSet():
self.db.expire_all()
for node_db in self.db.query(Node).filter(
# nodes may become unresponsive while provisioning
not_(Node.status == 'provisioning')
):
timedelta = (datetime.now() - node_db.timestamp).seconds
if timedelta > self.timeout:
logger.warning(
u"Node '{0}' seems to be offline "
"for {1} seconds...".format(
node_db.name,
timedelta
)
)
if node_db.online:
node_db.online = False
self.db.add(node_db)
self.db.commit()
notifier.notify(
"error",
u"Node '{0}' has gone away".format(
node_db.name or node_db.mac
),
node_id=node_db.id
)
self.sleep()
def set_proxy(proxy):
"""Replace http_proxy environment var for the scope of context execution
After exit from context old proxy value (if any) is restored
:param proxy: - proxy url
"""
variable_values = {
'http_proxy': os.environ.get('http_proxy'),
'https_proxy': os.environ.get('https_proxy')
}
for variable_name, variable_value in variable_values.items():
if variable_value:
logger.warning("{0} variable is already set with "
"value: {1}. Changing to {2}. Old value "
"will be restored after exit from script's "
"execution context"
.format(variable_name, variable_value, proxy))
os.environ[variable_name] = proxy
try:
yield
finally:
for variable_name, variable_value in variable_values.items():
if variable_value:
logger.info("Restoring old value for http_proxy")
os.environ[variable_name] = variable_value
else:
logger.info("Deleting set {0} environment variable"
.format(variable_name))
del os.environ[variable_name]
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning(
'Plugin with id "%s" is not found, skip it', plugin_id)
continue
enabled = container['enabled']\
and plugin_id == container['chosen_id']
ClusterPlugin.set_attributes(
cluster.id, plugin.id, enabled=enabled,
attrs=attrs if enabled or default else None
)
def update_by_agent(cls, instance, data):
"""Update Node instance with some specific cases for agent.
* don't update provisioning or error state back to discover
* don't update volume information if disks arrays is empty
:param data: dictionary of key-value pairs as object fields
:returns: Node instance
"""
# don't update provisioning and error back to discover
if instance.status in ('provisioning', 'error'):
if data.get('status', 'discover') == 'discover':
logger.debug(u"Node {0} has provisioning or error status - "
u"status not updated by agent".format(
instance.human_readable_name))
data['status'] = instance.status
# don't update volume information, if agent has sent an empty array
meta = data.get('meta', {})
if meta and len(meta.get('disks', [])) == 0 \
and instance.meta.get('disks'):
logger.warning(u'Node {0} has received an empty disks array - '
u'volume information will not be updated'.format(
instance.human_readable_name))
meta['disks'] = instance.meta['disks']
return cls.update(instance, data)
def update_roles(cls, instance, new_roles):
"""Update roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_roles: list of new role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(
u"Attempting to assign roles to node "
u"'{0}' which isn't added to cluster".format(
instance.name or instance.id
)
)
return
if new_roles:
instance.role_list = db().query(models.Role).filter_by(
release_id=instance.cluster.release_id,
).filter(
models.Role.name.in_(new_roles)
).all()
else:
instance.role_list = []
db().flush()
db().refresh(instance)
def PUT(self, node_id):
""":returns: JSONized Node object.
:http: * 200 (OK)
* 400 (invalid node data specified)
* 404 (node not found in db)
"""
node = self.get_object_or_404(Node, node_id)
if not node.attributes:
node.attributes = NodeAttributes(node_id=node.id)
data = self.checked_data(self.validator.validate_update)
network_manager = NetworkManager()
old_cluster_id = node.cluster_id
if data.get("pending_roles") == [] and node.cluster:
node.cluster.clear_pending_changes(node_id=node.id)
if "cluster_id" in data:
if data["cluster_id"] is None and node.cluster:
node.cluster.clear_pending_changes(node_id=node.id)
node.roles = node.pending_roles = []
node.cluster_id = data["cluster_id"]
if node.cluster_id != old_cluster_id:
if old_cluster_id:
network_manager.clear_assigned_networks(node)
network_manager.clear_all_allowed_networks(node.id)
if node.cluster_id:
network_manager.assign_networks_by_default(node)
network_manager.allow_network_assignment_to_all_interfaces(
node)
regenerate_volumes = any(
('roles' in data
and set(data['roles']) != set(node.roles), 'pending_roles' in data
and set(data['pending_roles']) != set(node.pending_roles),
node.cluster_id != old_cluster_id))
for key, value in data.iteritems():
# we don't allow to update id explicitly
# and updated cluster_id before all other fields
if key in ("id", "cluster_id"):
continue
setattr(node, key, value)
if not node.status in ('provisioning',
'deploying') and regenerate_volumes:
try:
node.attributes.volumes = \
node.volume_manager.gen_volumes_info()
except Exception as exc:
msg = (u"Failed to generate volumes "
"info for node '{0}': '{1}'").format(
node.name or data.get("mac") or data.get("id"),
str(exc) or "see logs for details")
logger.warning(traceback.format_exc())
notifier.notify("error", msg, node_id=node.id)
db().commit()
return self.render(node)
def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
variable_values = {
'http_proxy': os.environ.get('http_proxy'),
'https_proxy': os.environ.get('https_proxy')
}
for variable_name, variable_value in variable_values.items():
if variable_value:
logger.warning("{0} variable is already set with "
"value: {1}. Changing to {2}. Old value "
"will be restored after exit from script's "
"execution context"
.format(variable_name, variable_value, proxy))
os.environ[variable_name] = proxy
try:
yield
finally:
for variable_name, variable_value in variable_values.items():
if variable_value:
logger.info("Restoring old value for http_proxy")
os.environ[variable_name] = variable_value
else:
logger.info("Deleting set {0} environment variable"
.format(variable_name))
del os.environ[variable_name]
def cast(name, message, service=False):
logger.debug(
"RPC cast to orchestrator:\n{0}".format(
jsonutils.dumps(message, indent=4)
)
)
#测试使用
file_object = open('/opt/queuemsg.txt', 'w')
file_object.write(jsonutils.dumps(message, indent=4))
file_object.close()
use_queue = naily_queue if not service else naily_service_queue
use_exchange = naily_exchange if not service else naily_service_exchange
with Connection(conn_str) as conn:
with conn.Producer(serializer='json') as producer:
publish = functools.partial(producer.publish, message,
exchange=use_exchange, routing_key=name, declare=[use_queue])
try:
#pass
publish()
except amqp_exceptions.PreconditionFailed as e:
logger.warning(six.text_type(e))
# (dshulyak) we should drop both exchanges/queues in order
# for astute to be able to recover temporary queues
utils.delete_entities(
conn, naily_service_exchange, naily_service_queue,
naily_exchange, naily_queue)
publish()
def update_pending_roles(cls, instance, new_pending_roles):
"""Update pending_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_pending_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.name or instance.id))
return
logger.debug(u"Updating pending roles for node {0}: {1}".format(
instance.id, new_pending_roles))
if new_pending_roles == []:
instance.pending_role_list = []
#TODO(enchantner): research why the hell we need this
Cluster.clear_pending_changes(instance.cluster,
node_id=instance.id)
else:
instance.pending_role_list = db().query(models.Role).filter_by(
release_id=instance.cluster.release_id, ).filter(
models.Role.name.in_(new_pending_roles)).all()
db().flush()
db().refresh(instance)
def _update_dependencies(self):
"""Create dependencies that rely on regexp matching."""
for task in six.itervalues(self.node):
# tasks and groups should be used for declaring dependencies
# between tasks and roles (which are simply group of tasks)
available_groups = self.get_groups_subgraph().nodes()
for group in task.get('groups', ()):
pattern = NameMatchingPolicy.create(group)
not_matched = []
for available_group in available_groups:
if pattern.match(available_group):
self.add_edge(task['id'], available_group)
else:
not_matched.append(available_group)
# Add dependency for non-existing group which will be
# resolved in DeploymentGraphValidator
if len(available_groups) == len(not_matched):
self.add_edge(task['id'], group)
logger.warning('Group "%s" is an invalid dependency',
group)
available_groups = not_matched
for req in task.get('tasks', ()):
self.add_edge(req, task['id'])
def check_before_deployment(self, supertask):
# checking admin intersection with untagged
network_info = NetworkConfigurationSerializer\
.serialize_for_cluster(
self.cluster
)
check_networks = supertask.create_subtask('check_networks')
self._call_silently(check_networks,
tasks.CheckNetworksTask,
data=network_info,
check_admin_untagged=True)
db().refresh(check_networks)
if check_networks.status == 'error':
logger.warning("Checking networks failed: %s",
check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
db().delete(check_networks)
db().commit()
# checking prerequisites
check_before = supertask.create_subtask('check_before_deployment')
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
db().refresh(check_before)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == 'error':
logger.warning("Checking prerequisites failed: %s",
check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug(
"Checking prerequisites is successful, starting deployment...")
db().delete(check_before)
db().commit()
def POST(self, cluster_id):
""":returns: Http response.
:http: * 201 (nodes are successfully assigned)
* 400 (invalid nodes data specified)
"""
data = self.checked_data(self.validator.validate_collection_update,
cluster_id=cluster_id)
nodes = self.get_objects_list_or_404(Node, data.keys())
cluster = self.get_object_or_404(Cluster, cluster_id)
for node in nodes:
node.cluster = cluster
node.pending_roles = data[node.id]
node.pending_addition = True
try:
node.attributes.volumes = \
node.volume_manager.gen_volumes_info()
node.cluster.add_pending_changes("disks", node_id=node.id)
network_manager = node.cluster.network_manager
network_manager.assign_networks_by_default(node)
except Exception as exc:
logger.warning(traceback.format_exc())
notifier.notify(
"error",
u"Failed to generate attributes for node '{0}': '{1}'".
format(node.human_readable_name(),
str(exc) or u"see logs for details"),
node_id=node.id)
db().commit()
raise web.ok
def update_by_agent(cls, instance, data):
"""Update Node instance with some specific cases for agent.
* don't update provisioning or error state back to discover
* don't update volume information if disks arrays is empty
:param data: dictionary of key-value pairs as object fields
:returns: Node instance
"""
# don't update provisioning and error back to discover
data_status = data.get('status')
if instance.status in ('provisioning', 'error'):
if data.get('status', 'discover') == 'discover':
logger.debug(
u"Node {0} has provisioning or error status - "
u"status not updated by agent".format(
instance.human_readable_name
)
)
data.pop('status', None)
meta = data.get('meta', {})
# don't update volume information, if agent has sent an empty array
if len(meta.get('disks', [])) == 0 and instance.meta.get('disks'):
logger.warning(
u'Node {0} has received an empty disks array - '
u'volume information will not be updated'.format(
instance.human_readable_name
)
)
meta['disks'] = instance.meta['disks']
# don't update volume information, if it is locked by node status
if 'disks' in meta and cls.hardware_info_locked(instance):
logger.debug("Volume information is locked for update on node %s",
instance.human_readable_name)
meta['disks'] = instance.meta['disks']
if not cls.is_interfaces_configuration_locked(instance) \
and data.get('ip'):
if instance.cluster_id:
update_status = cls.check_ip_belongs_to_own_admin_network(
instance, data['ip'])
else:
update_status = cls.check_ip_belongs_to_any_admin_network(
instance, data['ip'])
if update_status:
if instance.status == consts.NODE_STATUSES.error and \
instance.error_type == consts.NODE_ERRORS.discover:
# accept the status from agent if the node had wrong IP
# previously
if data_status:
instance.status = data_status
else:
instance.status = consts.NODE_STATUSES.discover
else:
data.pop('status', None)
return cls.update(instance, data)
def GET(self):
""":returns: FUEL/FUELWeb commit SHA, release version.
:http: * 200 (OK)
"""
version = settings.VERSION
method = settings.AUTH['AUTHENTICATION_METHOD']
version['auth_required'] = method in ['fake', 'keystone']
version['release_versions'] = {}
for fl in glob.glob(self.release_versions):
with open(fl, "r") as release_yaml:
try:
version['release_versions'][
os.path.splitext(os.path.basename(fl))[0]
] = yaml.load(
release_yaml.read()
)
except Exception as exc:
logger.warning(
u"Failed to load release version "
"info from '{0}': {1}".format(
fl,
unicode(exc)
)
)
return version
def cast(name, message, service=False):
logger.debug("RPC cast to orchestrator:\n{0}".format(
jsonutils.dumps(message, indent=4)))
#测试使用
file_object = open('/opt/queuemsg.txt', 'w')
file_object.write(jsonutils.dumps(message, indent=4))
file_object.close()
use_queue = naily_queue if not service else naily_service_queue
use_exchange = naily_exchange if not service else naily_service_exchange
with Connection(conn_str) as conn:
with conn.Producer(serializer='json') as producer:
publish = functools.partial(producer.publish,
message,
exchange=use_exchange,
routing_key=name,
declare=[use_queue])
try:
#pass
publish()
except amqp_exceptions.PreconditionFailed as e:
logger.warning(six.text_type(e))
# (dshulyak) we should drop both exchanges/queues in order
# for astute to be able to recover temporary queues
utils.delete_entities(conn, naily_service_exchange,
naily_service_queue, naily_exchange,
naily_queue)
publish()
def PUT(self, node_id):
node = self.get_object_or_404(Node, node_id)
if not node.attributes:
node.attributes = NodeAttributes(node_id=node.id)
data = self.validator.validate_update(web.data())
for key, value in data.iteritems():
setattr(node, key, value)
if key == 'cluster_id':
if key:
self.allow_network_assignment_to_all_interfaces(node)
self.assign_networks_to_main_interface(node)
else:
self.clear_assigned_networks(node)
self.clear_all_allowed_networks(node)
if not node.status in ('provisioning', 'deploying') \
and "role" in data or "cluster_id" in data:
try:
node.attributes.volumes = \
node.volume_manager.gen_volumes_info()
except Exception as exc:
msg = (
u"Failed to generate volumes "
"info for node '{0}': '{1}'"
).format(
node.name or data.get("mac") or data.get("id"),
str(exc) or "see logs for details"
)
logger.warning(traceback.format_exc())
notifier.notify("error", msg, node_id=node.id)
self.db.commit()
return self.render(node)
def create_cluster(self, api=True, exclude=None, **kwargs):
cluster_data = {
'name': 'cluster-api-' + str(randint(0, 1000000)),
}
if kwargs:
cluster_data.update(kwargs)
if 'release_id' not in cluster_data:
cluster_data['release_id'] = self.create_release(api=False).id
if exclude and isinstance(exclude, list):
for ex in exclude:
try:
del cluster_data[ex]
except KeyError as err:
logger.warning(err)
if api:
resp = self.app.post(reverse('ClusterCollectionHandler'),
json.dumps(cluster_data),
headers=self.default_headers,
expect_errors=True)
self.tester.assertEquals(resp.status_code, 201)
cluster = json.loads(resp.body)
self.clusters.append(Cluster.get_by_uid(cluster['id']))
else:
cluster = Cluster.create(cluster_data)
db().commit()
self.clusters.append(cluster)
return cluster
def update_pending_roles(cls, instance, new_pending_roles):
if not instance.cluster_id:
logger.warning(
u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.name or instance.id
)
)
return
logger.debug(
u"Updating pending roles for node {0}: {1}".format(
instance.id,
new_pending_roles
)
)
if new_pending_roles == []:
instance.pending_role_list = []
# research why the hell we need this
Cluster.clear_pending_changes(
instance.cluster,
node_id=instance.id
)
else:
instance.pending_role_list = db().query(models.Role).filter_by(
release_id=instance.cluster.release_id,
).filter(
models.Role.name.in_(new_pending_roles)
).all()
db().flush()
db().refresh(instance)
def update_pending_roles(cls, instance, new_pending_roles):
"""Update pending_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_pending_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.full_name))
return
logger.debug(u"Updating pending roles for node {0}: {1}".format(
instance.full_name, new_pending_roles))
if new_pending_roles == []:
# TODO(enchantner): research why the hell we need this
Cluster.clear_pending_changes(instance.cluster,
node_id=instance.id)
instance.pending_roles = new_pending_roles
db().flush()
def check_before_deployment(self, supertask):
# checking admin intersection with untagged
network_info = self.serialize_network_cfg(self.cluster)
network_info["networks"] = [n for n in network_info["networks"] if n["name"] != "fuelweb_admin"]
check_networks = supertask.create_subtask(TASK_NAMES.check_networks)
self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_admin_untagged=True)
if check_networks.status == TASK_STATUSES.error:
logger.warning("Checking networks failed: %s", check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(TASK_NAMES.check_before_deployment)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == TASK_STATUSES.error:
logger.warning("Checking prerequisites failed: %s", check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug("Checking prerequisites is successful, starting deployment...")
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def _process_attr(cls, cluster, attr):
if not isinstance(attr, dict):
return
metadata = attr.get('metadata', {})
plugin_id = metadata.get('plugin_id')
if not plugin_id:
return
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
return
enabled = metadata.get('enabled', False)
# Value is true and plugin is not enabled for this cluster
# that means plugin was enabled on this request
if enabled and cluster not in plugin.clusters:
plugin.clusters.append(cluster)
# Value is false and plugin is enabled for this cluster
# that means plugin was disabled on this request
elif not enabled and cluster in plugin.clusters:
plugin.clusters.remove(cluster)
def text_format_safe(data, context):
try:
return data.format(**context)
except Exception as e:
logger.warning("Cannot format %s: %s. it will be used as is.",
data, six.text_type(e))
return data
def create_cluster(self, api=True, exclude=None, **kwargs):
cluster_data = {
'name': 'cluster-api-' + str(randint(0, 1000000)),
}
if kwargs:
cluster_data.update(kwargs)
if 'release_id' not in cluster_data:
cluster_data['release_id'] = self.create_release(api=False).id
if exclude and isinstance(exclude, list):
for ex in exclude:
try:
del cluster_data[ex]
except KeyError as err:
logger.warning(err)
if api:
resp = self.app.post(
reverse('ClusterCollectionHandler'),
jsonutils.dumps(cluster_data),
headers=self.default_headers,
expect_errors=True
)
self.tester.assertEqual(resp.status_code, 201)
cluster = resp.json_body
self.clusters.append(
Cluster.get_by_uid(cluster['id'])
)
else:
cluster = Cluster.create(cluster_data)
db().commit()
self.clusters.append(cluster)
return cluster
def _process_attr(cls, cluster, attr):
if not isinstance(attr, dict):
return
metadata = attr.get('metadata', {})
plugin_id = metadata.get('plugin_id')
if not plugin_id:
return
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
return
enabled = metadata.get('enabled', False)
# Value is true and plugin is not enabled for this cluster
# that means plugin was enabled on this request
if enabled and cluster not in plugin.clusters:
plugin.clusters.append(cluster)
# Value is false and plugin is enabled for this cluster
# that means plugin was disabled on this request
elif not enabled and cluster in plugin.clusters:
plugin.clusters.remove(cluster)
def update_primary_roles(cls, instance, new_primary_roles):
"""Update primary_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_primary_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.full_name))
return
assigned_roles = set(instance.roles + instance.pending_roles)
for role in new_primary_roles:
if role not in assigned_roles:
logger.warning(
u"Could not mark node {0} as primary for {1} role, "
u"because there's no assigned {1} role.".format(
instance.full_name, role))
return
logger.debug(u"Updating primary roles for node {0}: {1}".format(
instance.full_name, new_primary_roles))
instance.primary_roles = new_primary_roles
db().flush()
def get_nodes_by_role(cls, instance, role_name):
"""Get nodes related to some specific role
:param instance: cluster db object
:type: python object
:param role_name: node role name
:type: string
"""
role = db().query(models.Role).filter_by(
release_id=instance.release_id, name=role_name).first()
if not role:
logger.warning("%s role doesn't exist", role_name)
return []
nodes = db().query(models.Node).filter_by(cluster_id=instance.id)
deployed_nodes = nodes.join(
models.Node.role_list,
aliased=True).filter(models.Role.id == role.id).all()
pending_nodes = nodes.join(
models.Node.pending_role_list,
aliased=True).filter(models.Role.id == role.id).all()
return deployed_nodes + pending_nodes
def update_pending_roles(cls, instance, new_pending_roles):
"""Update pending_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_pending_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(
u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(instance.name or instance.id)
)
return
logger.debug(u"Updating pending roles for node {0}: {1}".format(instance.id, new_pending_roles))
if new_pending_roles == []:
instance.pending_role_list = []
# TODO(enchantner): research why the hell we need this
Cluster.clear_pending_changes(instance.cluster, node_id=instance.id)
else:
instance.pending_role_list = (
db()
.query(models.Role)
.filter_by(release_id=instance.cluster.release_id)
.filter(models.Role.name.in_(new_pending_roles))
.all()
)
db().flush()
db().refresh(instance)
def create_cluster(self, api=True, exclude=None, **kwargs):
cluster_data = {"name": "cluster-api-" + str(randint(0, 1000000))}
editable_attributes = kwargs.pop("editable_attributes", None)
if kwargs:
cluster_data.update(kwargs)
if "release_id" not in cluster_data:
cluster_data["release_id"] = self.create_release(api=False).id
if exclude and isinstance(exclude, list):
for ex in exclude:
try:
del cluster_data[ex]
except KeyError as err:
logger.warning(err)
if api:
resp = self.app.post(
reverse("ClusterCollectionHandler"),
jsonutils.dumps(cluster_data),
headers=self.default_headers,
expect_errors=True,
)
self.tester.assertEqual(resp.status_code, 201, resp.body)
cluster = resp.json_body
cluster_db = Cluster.get_by_uid(cluster["id"])
else:
cluster = Cluster.create(cluster_data)
cluster_db = cluster
db().commit()
self.clusters.append(cluster_db)
if editable_attributes:
Cluster.patch_attributes(cluster_db, {"editable": editable_attributes})
return cluster
def POST(self, cluster_id):
""":returns: Http response.
:http: * 201 (nodes are successfully assigned)
* 400 (invalid nodes data specified)
"""
data = self.checked_data(
self.validator.validate_collection_update,
cluster_id=cluster_id
)
nodes = self.get_objects_list_or_404(Node, data.keys())
cluster = self.get_object_or_404(Cluster, cluster_id)
for node in nodes:
node.cluster = cluster
node.pending_roles = data[node.id]
node.pending_addition = True
try:
node.attributes.volumes = \
node.volume_manager.gen_volumes_info()
node.cluster.add_pending_changes("disks", node_id=node.id)
network_manager = node.cluster.network_manager
network_manager.assign_networks_by_default(node)
except Exception as exc:
logger.warning(traceback.format_exc())
notifier.notify(
"error",
u"Failed to generate attributes for node '{0}': '{1}'"
.format(
node.human_readable_name(),
str(exc) or u"see logs for details"
),
node_id=node.id
)
db().commit()
raise web.ok
def text_format_safe(data, context):
try:
return data.format(**context)
except Exception as e:
logger.warning("Cannot format %s: %s. it will be used as is.", data,
six.text_type(e))
return data
def create_cluster(self, api=True, exclude=None, **kwargs):
cluster_data = {'name': 'cluster-api-' + str(randint(0, 1000000))}
if api:
cluster_data['release'] = self.create_release(api=False).id
else:
cluster_data['release'] = self.create_release(api=False)
if kwargs:
cluster_data.update(kwargs)
if exclude and isinstance(exclude, list):
for ex in exclude:
try:
del cluster_data[ex]
except KeyError as err:
logger.warning(err)
if api:
resp = self.app.post(reverse('ClusterCollectionHandler'),
json.dumps(cluster_data),
headers=self.default_headers)
self.tester.assertEquals(resp.status, 201)
cluster = json.loads(resp.body)
self.clusters.append(self.db.query(Cluster).get(cluster['id']))
else:
cluster = Cluster()
for field, value in cluster_data.iteritems():
setattr(cluster, field, value)
self.db.add(cluster)
self.db.commit()
self.clusters.append(cluster)
return cluster
def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
proxy_old_value = None
if os.environ.get("http_proxy"):
proxy_old_value = os.environ["http_proxy"]
logger.warning("http_proxy variable is already set with "
"value: {0}. Change to {1}. Old value "
"will be restored after exit from script's "
"execution context"
.format(proxy_old_value, proxy))
os.environ["http_proxy"] = proxy
try:
yield
except Exception as e:
logger.exception("Error while talking to proxy. Details: {0}"
.format(six.text_type(e)))
finally:
if proxy_old_value:
logger.info("Restoring old value for http_proxy")
os.environ["http_proxy"] = proxy_old_value
else:
logger.info("Deleting set http_proxy environment variable")
del os.environ["http_proxy"]
def set_proxy(proxy):
"""Replace http_proxy environment variable for the scope
of context execution. After exit from context old proxy value
(if any) is restored
:param proxy: - proxy url
"""
proxy_old_value = None
if os.environ.get("http_proxy"):
proxy_old_value = os.environ["http_proxy"]
logger.warning("http_proxy variable is already set with "
"value: {0}. Change to {1}. Old value "
"will be restored after exit from script's "
"execution context".format(proxy_old_value, proxy))
os.environ["http_proxy"] = proxy
try:
yield
except Exception as e:
logger.exception("Error while talking to proxy. Details: {0}".format(
six.text_type(e)))
finally:
if proxy_old_value:
logger.info("Restoring old value for http_proxy")
os.environ["http_proxy"] = proxy_old_value
else:
logger.info("Deleting set http_proxy environment variable")
del os.environ["http_proxy"]
def assign_network_to_interface(cls, instance, node):
"""Assign network to interface by default for single node
Assign given network to first available interface.
Checks interface type, if network is already assigned
and already assigned networks.
"""
if instance is None:
return
from nailgun import objects
untagged = cls.is_untagged(instance)
dedicated = instance.meta.get('dedicated_nic')
ifaces = objects.Node.get_interfaces_without_bonds_slaves(node)
for iface in ifaces:
if dedicated and iface.assigned_networks_list:
continue
for net in iface.assigned_networks_list:
if net.meta.get('dedicated_nic'):
break
if net == instance:
return
if untagged and cls.is_untagged(net):
break
else:
assigned_nets = iface.assigned_networks_list + [instance]
objects.NIC.assign_networks(iface, assigned_nets)
break
else:
logger.warning(
"Cannot assign network %r appropriately for "
"node %r. Set unassigned network to the "
"interface %r", instance.name, node.name, ifaces[0].name)
assigned_nets = ifaces[0].assigned_networks_list + [instance]
objects.NIC.assign_networks(ifaces[0], assigned_nets)
def update_roles(cls, instance, new_roles):
"""Update roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_roles: list of new role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(
u"Attempting to assign roles to node "
u"'{0}' which isn't added to cluster".format(
instance.name or instance.id
)
)
return
if new_roles:
instance.role_list = db().query(models.Role).filter_by(
release_id=instance.cluster.release_id,
).filter(
models.Role.name.in_(new_roles)
).all()
else:
instance.role_list = []
db().flush()
db().refresh(instance)
def update_by_agent(cls, instance, data):
"""Update Node instance with some specific cases for agent.
* don't update provisioning or error state back to discover
* don't update volume information if disks arrays is empty
:param data: dictionary of key-value pairs as object fields
:returns: Node instance
"""
# don't update provisioning and error back to discover
if instance.status in ('provisioning', 'error'):
if data.get('status', 'discover') == 'discover':
logger.debug(
u"Node {0} has provisioning or error status - "
u"status not updated by agent".format(
instance.human_readable_name
)
)
data['status'] = instance.status
# don't update volume information, if agent has sent an empty array
meta = data.get('meta', {})
if meta and len(meta.get('disks', [])) == 0 \
and instance.meta.get('disks'):
logger.warning(
u'Node {0} has received an empty disks array - '
u'volume information will not be updated'.format(
instance.human_readable_name
)
)
meta['disks'] = instance.meta['disks']
return cls.update(instance, data)
def update_volumes(cls, instance):
"""Update volumes for Node instance.
Adds pending "disks" changes for Cluster which Node belongs to
:param instance: Node instance
:returns: None
"""
attrs = instance.attributes
if not attrs:
attrs = cls.create_attributes(instance)
try:
attrs.volumes = instance.volume_manager.gen_volumes_info()
except Exception as exc:
msg = (u"Failed to generate volumes "
u"info for node '{0}': '{1}'").format(
instance.name or instance.mac or instance.id,
str(exc) or "see logs for details")
logger.warning(traceback.format_exc())
Notification.create({
"topic": "error",
"message": msg,
"node_id": instance.id
})
if instance.cluster_id:
Cluster.add_pending_changes(instance.cluster,
"disks",
node_id=instance.id)
db().add(attrs)
db().flush()
def _update_dependencies(self):
"""Create dependencies that rely on regexp matching."""
for task in six.itervalues(self.node):
# tasks and groups should be used for declaring dependencies
# between tasks and roles (which are simply group of tasks)
available_groups = self.get_groups_subgraph().nodes()
for group in task.get('groups', ()):
pattern = NameMatchingPolicy.create(group)
not_matched = []
for available_group in available_groups:
if pattern.match(available_group):
self.add_edge(task['id'], available_group)
else:
not_matched.append(available_group)
# Add dependency for non-existing group which will be
# resolved in DeploymentGraphValidator
if len(available_groups) == len(not_matched):
self.add_edge(task['id'], group)
logger.warning(
'Group "%s" is an invalid dependency', group)
available_groups = not_matched
for req in task.get('tasks', ()):
self.add_edge(req, task['id'])
def update_volumes(cls, instance):
attrs = instance.attributes
if not attrs:
attrs = cls.create_attributes(instance)
try:
attrs.volumes = instance.volume_manager.gen_volumes_info()
except Exception as exc:
msg = (
u"Failed to generate volumes "
u"info for node '{0}': '{1}'"
).format(
instance.name or instance.mac or instance.id,
str(exc) or "see logs for details"
)
logger.warning(traceback.format_exc())
Notification.create({
"topic": "error",
"message": msg,
"node_id": instance.id
})
if instance.cluster_id:
Cluster.add_pending_changes(
instance.cluster,
"disks",
node_id=instance.id
)
db().add(attrs)
db().flush()
def _get_pxe_iface_name(cls, node):
"""Returns appropriate pxe iface's name
In case when node has network scheme configured
we can not rely on its pxe interface calculation
algorithm anymore, because admin ip is moving to
bridge and 'pxe' property will have 'False' value
for all interfaces. In this case we should rely on
db where actual pxe interface was saved during the
bootstrap stage.
In case when node for some reason has no pxe interface
in db we should get pxe interface using appropriate
function `get_admin_physical_iface`.
"""
db_interfaces = node.nic_interfaces
pxe = next((
i for i in node.meta['interfaces'] if i.get('pxe') or
i.get('mac') == node.mac),
None)
if pxe:
return pxe.get('name')
pxe_db = next((i for i in db_interfaces if i.pxe), None)
if pxe_db:
return pxe_db.name
if db_interfaces:
return objects.Node.get_admin_physical_iface(node).name
logger.warning(u'Cannot find pxe interface for node "%s"',
node.full_name)
return None
def update_pending_roles(cls, instance, new_pending_roles):
"""Update pending_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_pending_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(
u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.full_name))
return
logger.debug(
u"Updating pending roles for node {0}: {1}".format(
instance.full_name,
new_pending_roles))
if new_pending_roles == []:
# TODO(enchantner): research why the hell we need this
Cluster.clear_pending_changes(
instance.cluster,
node_id=instance.id
)
instance.pending_roles = new_pending_roles
db().flush()
def stop_deployment_resp(cls, **kwargs):
logger.info("RPC method stop_deployment_resp received: %s" % jsonutils.dumps(kwargs))
task_uuid = kwargs.get("task_uuid")
nodes = kwargs.get("nodes", [])
ia_nodes = kwargs.get("inaccessible_nodes", [])
message = kwargs.get("error")
status = kwargs.get("status")
progress = kwargs.get("progress")
task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True)
stopping_task_names = [consts.TASK_NAMES.deploy, consts.TASK_NAMES.deployment, consts.TASK_NAMES.provision]
q_stop_tasks = objects.TaskCollection.filter_by_list(None, "name", stopping_task_names)
q_stop_tasks = objects.TaskCollection.filter_by(q_stop_tasks, cluster_id=task.cluster_id)
stop_tasks = objects.TaskCollection.order_by(q_stop_tasks, "id").all()
# Locking cluster
objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True)
if not stop_tasks:
logger.warning(
"stop_deployment_resp: deployment tasks \
not found for environment '%s'!",
task.cluster_id,
)
if status == consts.TASK_STATUSES.ready:
task.cluster.status = consts.CLUSTER_STATUSES.stopped
if stop_tasks:
map(db().delete, stop_tasks)
node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)]
q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids)
q_nodes = objects.NodeCollection.filter_by(q_nodes, cluster_id=task.cluster_id)
q_nodes = objects.NodeCollection.order_by(q_nodes, "id")
q_nodes = objects.NodeCollection.lock_for_update(q_nodes)
# locking Nodes for update
update_nodes = objects.NodeCollection.lock_for_update(q_nodes).all()
for node in update_nodes:
objects.Node.reset_to_discover(node)
if ia_nodes:
cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping")
message = (
u"Deployment of environment '{0}' was successfully stopped. "
u"Please make changes and reset the environment "
u"if you want to redeploy it.".format(task.cluster.name or task.cluster_id)
)
notifier.notify("done", message, task.cluster_id)
data = {"status": status, "progress": progress, "message": message}
objects.Task.update(task, data)
cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def update_primary_roles(cls, instance, new_primary_roles):
"""Update primary_roles for Node instance.
Logs an error if node doesn't belong to Cluster
:param instance: Node instance
:param new_primary_roles: list of new pending role names
:returns: None
"""
if not instance.cluster_id:
logger.warning(
u"Attempting to assign pending roles to node "
u"'{0}' which isn't added to cluster".format(
instance.full_name))
return
assigned_roles = set(instance.roles + instance.pending_roles)
for role in new_primary_roles:
if role not in assigned_roles:
logger.warning(
u"Could not mark node {0} as primary for {1} role, "
u"because there's no assigned {1} role.".format(
instance.full_name, role)
)
return
logger.debug(
u"Updating primary roles for node {0}: {1}".format(
instance.full_name,
new_primary_roles))
instance.primary_roles = new_primary_roles
db().flush()
def get_nodes_by_role(cls, instance, role_name):
"""Get nodes related to some specific role
:param instance: cluster db object
:type: python object
:param role_name: node role name
:type: string
"""
role = db().query(models.Role).filter_by(
release_id=instance.release_id, name=role_name).first()
if not role:
logger.warning("%s role doesn't exist", role_name)
return []
nodes = db().query(models.Node).filter_by(cluster_id=instance.id)
deployed_nodes = nodes.join(
models.Node.role_list, aliased=True).filter(
models.Role.id == role.id).all()
pending_nodes = nodes.join(
models.Node.pending_role_list, aliased=True).filter(
models.Role.id == role.id).all()
return deployed_nodes + pending_nodes
def POST(self):
data = self.checked_data()
node = Node()
for key, value in data.iteritems():
if key == "meta":
node.create_meta(value)
else:
setattr(node, key, value)
node.name = "Untitled (%s)" % data['mac'][-5:]
node.timestamp = datetime.now()
self.db.add(node)
self.db.commit()
node.attributes = NodeAttributes()
try:
node.attributes.volumes = node.volume_manager.gen_volumes_info()
if node.cluster:
node.cluster.add_pending_changes(
"disks",
node_id=node.id
)
except Exception as exc:
msg = (
u"Failed to generate volumes "
"info for node '{0}': '{1}'"
).format(
node.name or data.get("mac") or data.get("id"),
str(exc) or "see logs for details"
)
logger.warning(traceback.format_exc())
notifier.notify("error", msg, node_id=node.id)
self.db.add(node)
self.db.commit()
network_manager = NetworkManager()
# Add interfaces for node from 'meta'.
if node.meta and node.meta.get('interfaces'):
network_manager.update_interfaces_info(node.id)
if node.cluster_id:
network_manager.allow_network_assignment_to_all_interfaces(node.id)
network_manager.assign_networks_to_main_interface(node.id)
try:
ram = str(round(float(
node.meta['memory']['total']) / 1073741824, 1))
except (KeyError, TypeError, ValueError):
ram = "unknown"
cores = str(node.meta.get('cpu', {}).get('total', "unknown"))
notifier.notify("discover",
"New node with %s CPU core(s) "
"and %s GB memory is discovered" %
(cores, ram), node_id=node.id)
raise web.webapi.created(json.dumps(
NodeHandler.render(node),
indent=4
))
def __init__(cls, name, bases, dct):
super(HandlerRegistrator, cls).__init__(name, bases, dct)
if hasattr(cls, 'model'):
key = cls.model.__name__
if key in handlers:
logger.warning("Handler for %s already registered" % key)
return
handlers[key] = cls
def __init__(cls, name, bases, dct):
super(HandlerRegistrator, cls).__init__(name, bases, dct)
if hasattr(cls, 'model'):
key = cls.model.__name__
if key in handlers:
logger.warning("Handler for %s already registered" % key)
return
handlers[key] = cls
def check_before_deployment(self, supertask):
"""Performs checks before deployment
:param supertask: task SqlAlchemy object
"""
try:
# if there are VIPs with same names in the network configuration
# the error will be raised. Such situation may occur when, for
# example, enabled plugins contain conflicting network
# configuration
network_info = self.serialize_network_cfg(self.cluster)
except (errors.DuplicatedVIPNames, errors.NetworkRoleConflict) as e:
raise errors.CheckBeforeDeploymentError(e.message)
logger.info(u"Network info:\n{0}".format(
jsonutils.dumps(network_info, indent=4)))
# checking admin intersection with untagged
network_info["networks"] = [
n for n in network_info["networks"] if n["name"] != "fuelweb_admin"
]
check_networks = supertask.create_subtask(
consts.TASK_NAMES.check_networks)
self._call_silently(check_networks,
tasks.CheckNetworksTask,
data=network_info,
check_all_parameters=True)
if check_networks.status == consts.TASK_STATUSES.error:
logger.warning("Checking networks failed: %s",
check_networks.message)
raise errors.CheckBeforeDeploymentError(check_networks.message)
TaskHelper.set_ready_if_not_finished(check_networks)
db().delete(check_networks)
db().refresh(supertask)
db().flush()
# checking prerequisites
check_before = supertask.create_subtask(
consts.TASK_NAMES.check_before_deployment)
logger.debug("Checking prerequisites task: %s", check_before.uuid)
self._call_silently(check_before, tasks.CheckBeforeDeploymentTask)
# if failed to check prerequisites
# then task is already set to error
if check_before.status == consts.TASK_STATUSES.error:
logger.warning("Checking prerequisites failed: %s",
check_before.message)
raise errors.CheckBeforeDeploymentError(check_before.message)
logger.debug(
"Checking prerequisites is successful, starting deployment...")
TaskHelper.set_ready_if_not_finished(check_before)
db().delete(check_before)
db().refresh(supertask)
db().flush()
def stop_deployment_resp(cls, **kwargs):
logger.info("RPC method stop_deployment_resp received: %s" %
json.dumps(kwargs))
task_uuid = kwargs.get('task_uuid')
nodes = kwargs.get('nodes', [])
ia_nodes = kwargs.get('inaccessible_nodes', [])
message = kwargs.get('error')
status = kwargs.get('status')
progress = kwargs.get('progress')
task = TaskHelper.get_task_by_uuid(task_uuid)
stop_tasks = db().query(Task).filter_by(
cluster_id=task.cluster_id, ).filter(
Task.name.in_(["deploy", "deployment", "provision"])).all()
if not stop_tasks:
logger.warning(
"stop_deployment_resp: deployment tasks \
not found for environment '%s'!", task.cluster_id)
if status == "ready":
task.cluster.status = "stopped"
if stop_tasks:
map(db().delete, stop_tasks)
db().commit()
update_nodes = db().query(Node).filter(
Node.id.in_(
[n["uid"] for n in itertools.chain(nodes, ia_nodes)]),
Node.cluster_id == task.cluster_id).yield_per(100)
update_nodes.update(
{
"online": False,
"status": "discover",
"pending_addition": True
},
synchronize_session='fetch')
for n in update_nodes:
n.roles, n.pending_roles = n.pending_roles, n.roles
db().commit()
if ia_nodes:
cls._notify_inaccessible(task.cluster_id,
[n["uid"] for n in ia_nodes],
u"deployment stopping")
message = (u"Deployment of environment '{0}' "
u"was successfully stopped".format(task.cluster.name
or task.cluster_id))
notifier.notify("done", message, task.cluster_id)
TaskHelper.update_task_status(task_uuid, status, progress, message)
def PUT(self, node_id):
""":returns: JSONized Node object.
:http: * 200 (OK)
* 400 (invalid node data specified)
* 404 (node not found in db)
"""
node = self.get_object_or_404(Node, node_id)
if not node.attributes:
node.attributes = NodeAttributes(node_id=node.id)
data = self.checked_data(self.validator.validate_update)
network_manager = NetworkManager()
old_cluster_id = node.cluster_id
if data.get("pending_roles") == [] and node.cluster:
node.cluster.clear_pending_changes(node_id=node.id)
if "cluster_id" in data:
if data["cluster_id"] is None and node.cluster:
node.cluster.clear_pending_changes(node_id=node.id)
node.roles = node.pending_roles = []
node.cluster_id = data["cluster_id"]
if node.cluster_id != old_cluster_id:
if old_cluster_id:
network_manager.clear_assigned_networks(node)
network_manager.clear_all_allowed_networks(node.id)
if node.cluster_id:
network_manager.assign_networks_by_default(node)
network_manager.allow_network_assignment_to_all_interfaces(node)
regenerate_volumes = any(
(
"roles" in data and set(data["roles"]) != set(node.roles),
"pending_roles" in data and set(data["pending_roles"]) != set(node.pending_roles),
node.cluster_id != old_cluster_id,
)
)
for key, value in data.iteritems():
# we don't allow to update id explicitly
# and updated cluster_id before all other fields
if key in ("id", "cluster_id"):
continue
setattr(node, key, value)
if not node.status in ("provisioning", "deploying") and regenerate_volumes:
try:
node.attributes.volumes = node.volume_manager.gen_volumes_info()
except Exception as exc:
msg = (u"Failed to generate volumes " "info for node '{0}': '{1}'").format(
node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details"
)
logger.warning(traceback.format_exc())
notifier.notify("error", msg, node_id=node.id)
db().commit()
return self.render(node)
def POST(self):
data = self.validator.validate(web.data())
node = Node()
for key, value in data.iteritems():
setattr(node, key, value)
node.name = "Untitled (%s)" % data['mac'][-5:]
node.timestamp = datetime.now()
self.db.add(node)
self.db.commit()
node.attributes = NodeAttributes()
try:
node.attributes.volumes = node.volume_manager.gen_volumes_info()
if node.cluster:
node.cluster.add_pending_changes(
"disks",
node_id=node.id
)
except Exception as exc:
msg = (
u"Failed to generate volumes "
"info for node '{0}': '{1}'"
).format(
node.name or data.get("mac") or data.get("id"),
str(exc) or "see logs for details"
)
logger.warning(traceback.format_exc())
notifier.notify("error", msg, node_id=node.id)
self.db.add(node)
self.db.commit()
# Add interfaces for node from 'meta'.
if node.meta and node.meta.get('interfaces'):
nics = self.get_nics_from_meta(node)
map(self.db.add, nics)
self.db.commit()
if node.cluster_id:
self.allow_network_assignment_to_all_interfaces(node)
self.assign_networks_to_main_interface(node)
self.db.commit()
try:
ram = str(round(float(
node.meta['memory']['total']) / 1073741824, 1))
except (KeyError, TypeError, ValueError):
ram = "unknown"
cores = str(node.meta.get('cpu', {}).get('total', "unknown"))
notifier.notify("discover",
"New node with %s CPU core(s) "
"and %s GB memory is discovered" %
(cores, ram), node_id=node.id)
raise web.webapi.created(json.dumps(
NodeHandler.render(node),
indent=4
))
def set_default_attributes(cls, instance):
if not instance.cluster_id:
logger.warning(
u"Attempting to update attributes of node "
u"'{0}' which isn't added to any cluster".format(
instance.full_name))
return
instance.attributes = instance.cluster.release.node_attributes
def pending_roles(self, new_roles):
if not self.cluster:
logger.warning(u"Attempting to assign pending_roles to node "
u"'{0}' which isn't added to cluster".format(
self.name or self.id))
return
self.pending_role_list = db().query(Role).filter_by(
release_id=self.cluster.release_id, ).filter(
Role.name.in_(new_roles)).all()
def _update_nodes(transaction, nodes_instances, nodes_params):
allow_update = {
'name',
'status',
'hostname',
'kernel_params',
'pending_addition',
'pending_deletion',
'error_msg',
'online',
'progress',
}
# dry-run transactions must not update nodes except progress column
if transaction.dry_run:
allow_update = {'progress'}
for node in nodes_instances:
node_params = nodes_params.pop(node.uid)
for param in allow_update.intersection(node_params):
if param == 'status':
new_status = node_params['status']
if new_status == 'deleted':
# the deleted is special status which causes
# to delete node from cluster
objects.Node.remove_from_cluster(node)
elif new_status == 'error':
# TODO(bgaifullin) do not persist status in DB
node.status = new_status
node.error_type = node_params.get(
'error_type', consts.NODE_ERRORS.deploy)
node.progress = 100
# Notification on particular node failure
notifier.notify(consts.NOTIFICATION_TOPICS.error,
u"Node '{0}' failed: {1}".format(
node.name,
node_params.get(
'error_msg', "Unknown error")),
cluster_id=transaction.cluster_id,
node_id=node.uid,
task_uuid=transaction.uuid)
elif new_status == 'ready':
# TODO(bgaifullin) need to remove pengind roles concept
node.roles = list(set(node.roles + node.pending_roles))
node.pending_roles = []
node.progress = 100
node.status = new_status
else:
node.status = new_status
else:
setattr(node, param, node_params[param])
db.flush()
if nodes_params:
logger.warning("The following nodes are not found: %s",
",".join(sorted(nodes_params.keys())))
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
from nailgun.objects import Release
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
propagate_task_deploy = get_in(
attributes, 'common', 'propagate_task_deploy', 'value')
if propagate_task_deploy is not None:
legacy_tasks_are_ignored = not propagate_task_deploy
else:
legacy_tasks_are_ignored = not get_in(
cluster.attributes.editable,
'common', 'propagate_task_deploy', 'value')
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning(
'Plugin with id "%s" is not found, skip it', plugin_id)
continue
enabled = container['enabled'] \
and plugin_id == container['chosen_id']
if (enabled and
Release.is_lcm_supported(cluster.release) and
legacy_tasks_are_ignored and
cls.contains_legacy_tasks(
wrap_plugin(Plugin.get_by_uid(plugin.id)))):
raise errors.InvalidData(
'Cannot enable plugin with legacy tasks unless '
'propagate_task_deploy attribute is set. '
'Ensure tasks.yaml is empty and all tasks '
'has version >= 2.0.0.')
ClusterPlugin.set_attributes(
cluster.id, plugin.id, enabled=enabled,
attrs=attrs if enabled or default else None
)
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
from nailgun.objects import Release
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
propagate_task_deploy = get_in(attributes, 'common',
'propagate_task_deploy', 'value')
if propagate_task_deploy is not None:
legacy_tasks_are_ignored = not propagate_task_deploy
else:
legacy_tasks_are_ignored = not get_in(
cluster.attributes.editable, 'common', 'propagate_task_deploy',
'value')
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
continue
enabled = container['enabled'] \
and plugin_id == container['chosen_id']
if (enabled and Release.is_lcm_supported(cluster.release)
and legacy_tasks_are_ignored
and cls.contains_legacy_tasks(
wrap_plugin(Plugin.get_by_uid(plugin.id)))):
raise errors.InvalidData(
'Cannot enable plugin with legacy tasks unless '
'propagate_task_deploy attribute is set. '
'Ensure tasks.yaml is empty and all tasks '
'has version >= 2.0.0.')
ClusterPlugin.set_attributes(
cluster.id,
plugin.id,
enabled=enabled,
attrs=attrs if enabled or default else None)
def set_primary_role(cls, intance, nodes, role_name):
"""Method for assigning primary attribute for specific role.
- verify that there is no primary attribute of specific role
assigned to cluster nodes with this role in role list
or pending role list, and this node is not marked for deletion
- if there is no primary role assigned, filter nodes which have current
role in roles_list or pending_role_list
- if there is nodes with ready state - they should have higher priority
- if role was in primary_role_list - change primary attribute
for that association, same for role_list, this is required
because deployment_serializer used by cli to generate deployment info
:param instance: Cluster db objects
:param nodes: list of Node db objects
:param role_name: string with known role name
"""
all_roles = intance.release.role_list
role = next((r for r in all_roles if r.name == role_name), None)
if role is None:
logger.warning('Trying to assign primary for non-existing role %s',
role_name)
return
node = db().query(
models.Node).filter_by(pending_deletion=False).filter(
or_(
models.Node.role_associations.any(role=role.id,
primary=True),
models.Node.pending_role_associations.any(
role=role.id, primary=True))).filter(
models.Node.cluster == intance).first()
if not node:
filtered_nodes = []
for node in nodes:
if (not node.pending_deletion
and (role in node.role_list
or role in node.pending_role_list)):
filtered_nodes.append(node)
filtered_nodes = sorted(filtered_nodes, key=lambda node: node.id)
if filtered_nodes:
primary_node = next(
(node for node in filtered_nodes
if node.status == consts.NODE_STATUSES.ready),
filtered_nodes[0])
if role in primary_node.role_list:
associations = primary_node.role_associations
elif role in primary_node.pending_role_list:
associations = primary_node.pending_role_associations
else:
logger.warning(('Role %s neither in pending, nor role.'
' You hit strange bug'), role_name)
return
for assoc in associations:
if assoc.role == role.id:
assoc.primary = True
db().flush()
def update_by_agent(cls, instance, data):
"""Update Node instance with some specific cases for agent.
* don't update provisioning or error state back to discover
* don't update volume information if disks arrays is empty
:param data: dictionary of key-value pairs as object fields
:returns: Node instance
"""
# don't update provisioning and error back to discover
data_status = data.get('status')
if instance.status in ('provisioning', 'error'):
if data.get('status', 'discover') == 'discover':
logger.debug(u"Node {0} has provisioning or error status - "
u"status not updated by agent".format(
instance.human_readable_name))
data.pop('status', None)
meta = data.get('meta', {})
# don't update volume information, if agent has sent an empty array
if len(meta.get('disks', [])) == 0 and instance.meta.get('disks'):
logger.warning(u'Node {0} has received an empty disks array - '
u'volume information will not be updated'.format(
instance.human_readable_name))
meta['disks'] = instance.meta['disks']
# don't update volume information, if it is locked by node status
if 'disks' in meta and cls.hardware_info_locked(instance):
logger.debug("Volume information is locked for update on node %s",
instance.human_readable_name)
meta['disks'] = instance.meta['disks']
# (dshulyak) change this verification to NODE_STATUSES.deploying
# after we will reuse ips from dhcp range
if data.get('ip'):
if instance.cluster_id:
update_status = cls.check_ip_belongs_to_own_admin_network(
instance, data['ip'])
else:
update_status = cls.check_ip_belongs_to_any_admin_network(
instance, data['ip'])
if update_status:
if instance.status == consts.NODE_STATUSES.error and \
instance.error_type == consts.NODE_ERRORS.discover:
# accept the status from agent if the node had wrong IP
# previously
if data_status:
instance.status = data_status
else:
instance.status = consts.NODE_STATUSES.discover
else:
data.pop('status', None)
return cls.update(instance, data)
