def run(self): while not self.stoprequest.isSet(): self.db.expire_all() for node_db in self.db.query(Node).filter( # nodes may become unresponsive while provisioning not_(Node.status == 'provisioning') ): timedelta = (datetime.now() - node_db.timestamp).seconds if timedelta > self.timeout: logger.warning( u"Node '{0}' seems to be offline " "for {1} seconds...".format( node_db.name, timedelta ) ) if node_db.online: node_db.online = False self.db.add(node_db) self.db.commit() notifier.notify( "error", u"Node '{0}' has gone away".format( node_db.name or node_db.mac ), node_id=node_db.id ) self.sleep()
def set_proxy(proxy): """Replace http_proxy environment var for the scope of context execution After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ variable_values = { 'http_proxy': os.environ.get('http_proxy'), 'https_proxy': os.environ.get('https_proxy') } for variable_name, variable_value in variable_values.items(): if variable_value: logger.warning("{0} variable is already set with " "value: {1}. Changing to {2}. Old value " "will be restored after exit from script's " "execution context" .format(variable_name, variable_value, proxy)) os.environ[variable_name] = proxy try: yield finally: for variable_name, variable_value in variable_values.items(): if variable_value: logger.info("Restoring old value for http_proxy") os.environ[variable_name] = variable_value else: logger.info("Deleting set {0} environment variable" .format(variable_name)) del os.environ[variable_name]
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning( 'Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled']\ and plugin_id == container['chosen_id'] ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None )
def update_by_agent(cls, instance, data): """Update Node instance with some specific cases for agent. * don't update provisioning or error state back to discover * don't update volume information if disks arrays is empty :param data: dictionary of key-value pairs as object fields :returns: Node instance """ # don't update provisioning and error back to discover if instance.status in ('provisioning', 'error'): if data.get('status', 'discover') == 'discover': logger.debug(u"Node {0} has provisioning or error status - " u"status not updated by agent".format( instance.human_readable_name)) data['status'] = instance.status # don't update volume information, if agent has sent an empty array meta = data.get('meta', {}) if meta and len(meta.get('disks', [])) == 0 \ and instance.meta.get('disks'): logger.warning(u'Node {0} has received an empty disks array - ' u'volume information will not be updated'.format( instance.human_readable_name)) meta['disks'] = instance.meta['disks'] return cls.update(instance, data)
def update_roles(cls, instance, new_roles): """Update roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_roles: list of new role names :returns: None """ if not instance.cluster_id: logger.warning( u"Attempting to assign roles to node " u"'{0}' which isn't added to cluster".format( instance.name or instance.id ) ) return if new_roles: instance.role_list = db().query(models.Role).filter_by( release_id=instance.cluster.release_id, ).filter( models.Role.name.in_(new_roles) ).all() else: instance.role_list = [] db().flush() db().refresh(instance)
def PUT(self, node_id): """:returns: JSONized Node object. :http: * 200 (OK) * 400 (invalid node data specified) * 404 (node not found in db) """ node = self.get_object_or_404(Node, node_id) if not node.attributes: node.attributes = NodeAttributes(node_id=node.id) data = self.checked_data(self.validator.validate_update) network_manager = NetworkManager() old_cluster_id = node.cluster_id if data.get("pending_roles") == [] and node.cluster: node.cluster.clear_pending_changes(node_id=node.id) if "cluster_id" in data: if data["cluster_id"] is None and node.cluster: node.cluster.clear_pending_changes(node_id=node.id) node.roles = node.pending_roles = [] node.cluster_id = data["cluster_id"] if node.cluster_id != old_cluster_id: if old_cluster_id: network_manager.clear_assigned_networks(node) network_manager.clear_all_allowed_networks(node.id) if node.cluster_id: network_manager.assign_networks_by_default(node) network_manager.allow_network_assignment_to_all_interfaces( node) regenerate_volumes = any( ('roles' in data and set(data['roles']) != set(node.roles), 'pending_roles' in data and set(data['pending_roles']) != set(node.pending_roles), node.cluster_id != old_cluster_id)) for key, value in data.iteritems(): # we don't allow to update id explicitly # and updated cluster_id before all other fields if key in ("id", "cluster_id"): continue setattr(node, key, value) if not node.status in ('provisioning', 'deploying') and regenerate_volumes: try: node.attributes.volumes = \ node.volume_manager.gen_volumes_info() except Exception as exc: msg = (u"Failed to generate volumes " "info for node '{0}': '{1}'").format( node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details") logger.warning(traceback.format_exc()) notifier.notify("error", msg, node_id=node.id) db().commit() return self.render(node)
def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ variable_values = { 'http_proxy': os.environ.get('http_proxy'), 'https_proxy': os.environ.get('https_proxy') } for variable_name, variable_value in variable_values.items(): if variable_value: logger.warning("{0} variable is already set with " "value: {1}. Changing to {2}. Old value " "will be restored after exit from script's " "execution context" .format(variable_name, variable_value, proxy)) os.environ[variable_name] = proxy try: yield finally: for variable_name, variable_value in variable_values.items(): if variable_value: logger.info("Restoring old value for http_proxy") os.environ[variable_name] = variable_value else: logger.info("Deleting set {0} environment variable" .format(variable_name)) del os.environ[variable_name]
def cast(name, message, service=False): logger.debug( "RPC cast to orchestrator:\n{0}".format( jsonutils.dumps(message, indent=4) ) ) #测试使用 file_object = open('/opt/queuemsg.txt', 'w') file_object.write(jsonutils.dumps(message, indent=4)) file_object.close() use_queue = naily_queue if not service else naily_service_queue use_exchange = naily_exchange if not service else naily_service_exchange with Connection(conn_str) as conn: with conn.Producer(serializer='json') as producer: publish = functools.partial(producer.publish, message, exchange=use_exchange, routing_key=name, declare=[use_queue]) try: #pass publish() except amqp_exceptions.PreconditionFailed as e: logger.warning(six.text_type(e)) # (dshulyak) we should drop both exchanges/queues in order # for astute to be able to recover temporary queues utils.delete_entities( conn, naily_service_exchange, naily_service_queue, naily_exchange, naily_queue) publish()
def update_pending_roles(cls, instance, new_pending_roles): """Update pending_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_pending_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning(u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.name or instance.id)) return logger.debug(u"Updating pending roles for node {0}: {1}".format( instance.id, new_pending_roles)) if new_pending_roles == []: instance.pending_role_list = [] #TODO(enchantner): research why the hell we need this Cluster.clear_pending_changes(instance.cluster, node_id=instance.id) else: instance.pending_role_list = db().query(models.Role).filter_by( release_id=instance.cluster.release_id, ).filter( models.Role.name.in_(new_pending_roles)).all() db().flush() db().refresh(instance)
def _update_dependencies(self): """Create dependencies that rely on regexp matching.""" for task in six.itervalues(self.node): # tasks and groups should be used for declaring dependencies # between tasks and roles (which are simply group of tasks) available_groups = self.get_groups_subgraph().nodes() for group in task.get('groups', ()): pattern = NameMatchingPolicy.create(group) not_matched = [] for available_group in available_groups: if pattern.match(available_group): self.add_edge(task['id'], available_group) else: not_matched.append(available_group) # Add dependency for non-existing group which will be # resolved in DeploymentGraphValidator if len(available_groups) == len(not_matched): self.add_edge(task['id'], group) logger.warning('Group "%s" is an invalid dependency', group) available_groups = not_matched for req in task.get('tasks', ()): self.add_edge(req, task['id'])
def check_before_deployment(self, supertask): # checking admin intersection with untagged network_info = NetworkConfigurationSerializer\ .serialize_for_cluster( self.cluster ) check_networks = supertask.create_subtask('check_networks') self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_admin_untagged=True) db().refresh(check_networks) if check_networks.status == 'error': logger.warning("Checking networks failed: %s", check_networks.message) raise errors.CheckBeforeDeploymentError(check_networks.message) db().delete(check_networks) db().commit() # checking prerequisites check_before = supertask.create_subtask('check_before_deployment') logger.debug("Checking prerequisites task: %s", check_before.uuid) self._call_silently(check_before, tasks.CheckBeforeDeploymentTask) db().refresh(check_before) # if failed to check prerequisites # then task is already set to error if check_before.status == 'error': logger.warning("Checking prerequisites failed: %s", check_before.message) raise errors.CheckBeforeDeploymentError(check_before.message) logger.debug( "Checking prerequisites is successful, starting deployment...") db().delete(check_before) db().commit()
def POST(self, cluster_id): """:returns: Http response. :http: * 201 (nodes are successfully assigned) * 400 (invalid nodes data specified) """ data = self.checked_data(self.validator.validate_collection_update, cluster_id=cluster_id) nodes = self.get_objects_list_or_404(Node, data.keys()) cluster = self.get_object_or_404(Cluster, cluster_id) for node in nodes: node.cluster = cluster node.pending_roles = data[node.id] node.pending_addition = True try: node.attributes.volumes = \ node.volume_manager.gen_volumes_info() node.cluster.add_pending_changes("disks", node_id=node.id) network_manager = node.cluster.network_manager network_manager.assign_networks_by_default(node) except Exception as exc: logger.warning(traceback.format_exc()) notifier.notify( "error", u"Failed to generate attributes for node '{0}': '{1}'". format(node.human_readable_name(), str(exc) or u"see logs for details"), node_id=node.id) db().commit() raise web.ok
def update_by_agent(cls, instance, data): """Update Node instance with some specific cases for agent. * don't update provisioning or error state back to discover * don't update volume information if disks arrays is empty :param data: dictionary of key-value pairs as object fields :returns: Node instance """ # don't update provisioning and error back to discover data_status = data.get('status') if instance.status in ('provisioning', 'error'): if data.get('status', 'discover') == 'discover': logger.debug( u"Node {0} has provisioning or error status - " u"status not updated by agent".format( instance.human_readable_name ) ) data.pop('status', None) meta = data.get('meta', {}) # don't update volume information, if agent has sent an empty array if len(meta.get('disks', [])) == 0 and instance.meta.get('disks'): logger.warning( u'Node {0} has received an empty disks array - ' u'volume information will not be updated'.format( instance.human_readable_name ) ) meta['disks'] = instance.meta['disks'] # don't update volume information, if it is locked by node status if 'disks' in meta and cls.hardware_info_locked(instance): logger.debug("Volume information is locked for update on node %s", instance.human_readable_name) meta['disks'] = instance.meta['disks'] if not cls.is_interfaces_configuration_locked(instance) \ and data.get('ip'): if instance.cluster_id: update_status = cls.check_ip_belongs_to_own_admin_network( instance, data['ip']) else: update_status = cls.check_ip_belongs_to_any_admin_network( instance, data['ip']) if update_status: if instance.status == consts.NODE_STATUSES.error and \ instance.error_type == consts.NODE_ERRORS.discover: # accept the status from agent if the node had wrong IP # previously if data_status: instance.status = data_status else: instance.status = consts.NODE_STATUSES.discover else: data.pop('status', None) return cls.update(instance, data)
def GET(self): """:returns: FUEL/FUELWeb commit SHA, release version. :http: * 200 (OK) """ version = settings.VERSION method = settings.AUTH['AUTHENTICATION_METHOD'] version['auth_required'] = method in ['fake', 'keystone'] version['release_versions'] = {} for fl in glob.glob(self.release_versions): with open(fl, "r") as release_yaml: try: version['release_versions'][ os.path.splitext(os.path.basename(fl))[0] ] = yaml.load( release_yaml.read() ) except Exception as exc: logger.warning( u"Failed to load release version " "info from '{0}': {1}".format( fl, unicode(exc) ) ) return version
def cast(name, message, service=False): logger.debug("RPC cast to orchestrator:\n{0}".format( jsonutils.dumps(message, indent=4))) #测试使用 file_object = open('/opt/queuemsg.txt', 'w') file_object.write(jsonutils.dumps(message, indent=4)) file_object.close() use_queue = naily_queue if not service else naily_service_queue use_exchange = naily_exchange if not service else naily_service_exchange with Connection(conn_str) as conn: with conn.Producer(serializer='json') as producer: publish = functools.partial(producer.publish, message, exchange=use_exchange, routing_key=name, declare=[use_queue]) try: #pass publish() except amqp_exceptions.PreconditionFailed as e: logger.warning(six.text_type(e)) # (dshulyak) we should drop both exchanges/queues in order # for astute to be able to recover temporary queues utils.delete_entities(conn, naily_service_exchange, naily_service_queue, naily_exchange, naily_queue) publish()
def PUT(self, node_id): node = self.get_object_or_404(Node, node_id) if not node.attributes: node.attributes = NodeAttributes(node_id=node.id) data = self.validator.validate_update(web.data()) for key, value in data.iteritems(): setattr(node, key, value) if key == 'cluster_id': if key: self.allow_network_assignment_to_all_interfaces(node) self.assign_networks_to_main_interface(node) else: self.clear_assigned_networks(node) self.clear_all_allowed_networks(node) if not node.status in ('provisioning', 'deploying') \ and "role" in data or "cluster_id" in data: try: node.attributes.volumes = \ node.volume_manager.gen_volumes_info() except Exception as exc: msg = ( u"Failed to generate volumes " "info for node '{0}': '{1}'" ).format( node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details" ) logger.warning(traceback.format_exc()) notifier.notify("error", msg, node_id=node.id) self.db.commit() return self.render(node)
def create_cluster(self, api=True, exclude=None, **kwargs): cluster_data = { 'name': 'cluster-api-' + str(randint(0, 1000000)), } if kwargs: cluster_data.update(kwargs) if 'release_id' not in cluster_data: cluster_data['release_id'] = self.create_release(api=False).id if exclude and isinstance(exclude, list): for ex in exclude: try: del cluster_data[ex] except KeyError as err: logger.warning(err) if api: resp = self.app.post(reverse('ClusterCollectionHandler'), json.dumps(cluster_data), headers=self.default_headers, expect_errors=True) self.tester.assertEquals(resp.status_code, 201) cluster = json.loads(resp.body) self.clusters.append(Cluster.get_by_uid(cluster['id'])) else: cluster = Cluster.create(cluster_data) db().commit() self.clusters.append(cluster) return cluster
def update_pending_roles(cls, instance, new_pending_roles): if not instance.cluster_id: logger.warning( u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.name or instance.id ) ) return logger.debug( u"Updating pending roles for node {0}: {1}".format( instance.id, new_pending_roles ) ) if new_pending_roles == []: instance.pending_role_list = [] # research why the hell we need this Cluster.clear_pending_changes( instance.cluster, node_id=instance.id ) else: instance.pending_role_list = db().query(models.Role).filter_by( release_id=instance.cluster.release_id, ).filter( models.Role.name.in_(new_pending_roles) ).all() db().flush() db().refresh(instance)
def update_pending_roles(cls, instance, new_pending_roles): """Update pending_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_pending_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning(u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.full_name)) return logger.debug(u"Updating pending roles for node {0}: {1}".format( instance.full_name, new_pending_roles)) if new_pending_roles == []: # TODO(enchantner): research why the hell we need this Cluster.clear_pending_changes(instance.cluster, node_id=instance.id) instance.pending_roles = new_pending_roles db().flush()
def check_before_deployment(self, supertask): # checking admin intersection with untagged network_info = self.serialize_network_cfg(self.cluster) network_info["networks"] = [n for n in network_info["networks"] if n["name"] != "fuelweb_admin"] check_networks = supertask.create_subtask(TASK_NAMES.check_networks) self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_admin_untagged=True) if check_networks.status == TASK_STATUSES.error: logger.warning("Checking networks failed: %s", check_networks.message) raise errors.CheckBeforeDeploymentError(check_networks.message) TaskHelper.set_ready_if_not_finished(check_networks) db().delete(check_networks) db().refresh(supertask) db().flush() # checking prerequisites check_before = supertask.create_subtask(TASK_NAMES.check_before_deployment) logger.debug("Checking prerequisites task: %s", check_before.uuid) self._call_silently(check_before, tasks.CheckBeforeDeploymentTask) # if failed to check prerequisites # then task is already set to error if check_before.status == TASK_STATUSES.error: logger.warning("Checking prerequisites failed: %s", check_before.message) raise errors.CheckBeforeDeploymentError(check_before.message) logger.debug("Checking prerequisites is successful, starting deployment...") TaskHelper.set_ready_if_not_finished(check_before) db().delete(check_before) db().refresh(supertask) db().flush()
def _process_attr(cls, cluster, attr): if not isinstance(attr, dict): return metadata = attr.get('metadata', {}) plugin_id = metadata.get('plugin_id') if not plugin_id: return plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) return enabled = metadata.get('enabled', False) # Value is true and plugin is not enabled for this cluster # that means plugin was enabled on this request if enabled and cluster not in plugin.clusters: plugin.clusters.append(cluster) # Value is false and plugin is enabled for this cluster # that means plugin was disabled on this request elif not enabled and cluster in plugin.clusters: plugin.clusters.remove(cluster)
def text_format_safe(data, context): try: return data.format(**context) except Exception as e: logger.warning("Cannot format %s: %s. it will be used as is.", data, six.text_type(e)) return data
def create_cluster(self, api=True, exclude=None, **kwargs): cluster_data = { 'name': 'cluster-api-' + str(randint(0, 1000000)), } if kwargs: cluster_data.update(kwargs) if 'release_id' not in cluster_data: cluster_data['release_id'] = self.create_release(api=False).id if exclude and isinstance(exclude, list): for ex in exclude: try: del cluster_data[ex] except KeyError as err: logger.warning(err) if api: resp = self.app.post( reverse('ClusterCollectionHandler'), jsonutils.dumps(cluster_data), headers=self.default_headers, expect_errors=True ) self.tester.assertEqual(resp.status_code, 201) cluster = resp.json_body self.clusters.append( Cluster.get_by_uid(cluster['id']) ) else: cluster = Cluster.create(cluster_data) db().commit() self.clusters.append(cluster) return cluster
def _process_attr(cls, cluster, attr): if not isinstance(attr, dict): return metadata = attr.get('metadata', {}) plugin_id = metadata.get('plugin_id') if not plugin_id: return plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) return enabled = metadata.get('enabled', False) # Value is true and plugin is not enabled for this cluster # that means plugin was enabled on this request if enabled and cluster not in plugin.clusters: plugin.clusters.append(cluster) # Value is false and plugin is enabled for this cluster # that means plugin was disabled on this request elif not enabled and cluster in plugin.clusters: plugin.clusters.remove(cluster)
def update_primary_roles(cls, instance, new_primary_roles): """Update primary_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_primary_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning(u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.full_name)) return assigned_roles = set(instance.roles + instance.pending_roles) for role in new_primary_roles: if role not in assigned_roles: logger.warning( u"Could not mark node {0} as primary for {1} role, " u"because there's no assigned {1} role.".format( instance.full_name, role)) return logger.debug(u"Updating primary roles for node {0}: {1}".format( instance.full_name, new_primary_roles)) instance.primary_roles = new_primary_roles db().flush()
def get_nodes_by_role(cls, instance, role_name): """Get nodes related to some specific role :param instance: cluster db object :type: python object :param role_name: node role name :type: string """ role = db().query(models.Role).filter_by( release_id=instance.release_id, name=role_name).first() if not role: logger.warning("%s role doesn't exist", role_name) return [] nodes = db().query(models.Node).filter_by(cluster_id=instance.id) deployed_nodes = nodes.join( models.Node.role_list, aliased=True).filter(models.Role.id == role.id).all() pending_nodes = nodes.join( models.Node.pending_role_list, aliased=True).filter(models.Role.id == role.id).all() return deployed_nodes + pending_nodes
def update_pending_roles(cls, instance, new_pending_roles): """Update pending_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_pending_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning( u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format(instance.name or instance.id) ) return logger.debug(u"Updating pending roles for node {0}: {1}".format(instance.id, new_pending_roles)) if new_pending_roles == []: instance.pending_role_list = [] # TODO(enchantner): research why the hell we need this Cluster.clear_pending_changes(instance.cluster, node_id=instance.id) else: instance.pending_role_list = ( db() .query(models.Role) .filter_by(release_id=instance.cluster.release_id) .filter(models.Role.name.in_(new_pending_roles)) .all() ) db().flush() db().refresh(instance)
def create_cluster(self, api=True, exclude=None, **kwargs): cluster_data = {"name": "cluster-api-" + str(randint(0, 1000000))} editable_attributes = kwargs.pop("editable_attributes", None) if kwargs: cluster_data.update(kwargs) if "release_id" not in cluster_data: cluster_data["release_id"] = self.create_release(api=False).id if exclude and isinstance(exclude, list): for ex in exclude: try: del cluster_data[ex] except KeyError as err: logger.warning(err) if api: resp = self.app.post( reverse("ClusterCollectionHandler"), jsonutils.dumps(cluster_data), headers=self.default_headers, expect_errors=True, ) self.tester.assertEqual(resp.status_code, 201, resp.body) cluster = resp.json_body cluster_db = Cluster.get_by_uid(cluster["id"]) else: cluster = Cluster.create(cluster_data) cluster_db = cluster db().commit() self.clusters.append(cluster_db) if editable_attributes: Cluster.patch_attributes(cluster_db, {"editable": editable_attributes}) return cluster
def POST(self, cluster_id): """:returns: Http response. :http: * 201 (nodes are successfully assigned) * 400 (invalid nodes data specified) """ data = self.checked_data( self.validator.validate_collection_update, cluster_id=cluster_id ) nodes = self.get_objects_list_or_404(Node, data.keys()) cluster = self.get_object_or_404(Cluster, cluster_id) for node in nodes: node.cluster = cluster node.pending_roles = data[node.id] node.pending_addition = True try: node.attributes.volumes = \ node.volume_manager.gen_volumes_info() node.cluster.add_pending_changes("disks", node_id=node.id) network_manager = node.cluster.network_manager network_manager.assign_networks_by_default(node) except Exception as exc: logger.warning(traceback.format_exc()) notifier.notify( "error", u"Failed to generate attributes for node '{0}': '{1}'" .format( node.human_readable_name(), str(exc) or u"see logs for details" ), node_id=node.id ) db().commit() raise web.ok
def text_format_safe(data, context): try: return data.format(**context) except Exception as e: logger.warning("Cannot format %s: %s. it will be used as is.", data, six.text_type(e)) return data
def create_cluster(self, api=True, exclude=None, **kwargs): cluster_data = {'name': 'cluster-api-' + str(randint(0, 1000000))} if api: cluster_data['release'] = self.create_release(api=False).id else: cluster_data['release'] = self.create_release(api=False) if kwargs: cluster_data.update(kwargs) if exclude and isinstance(exclude, list): for ex in exclude: try: del cluster_data[ex] except KeyError as err: logger.warning(err) if api: resp = self.app.post(reverse('ClusterCollectionHandler'), json.dumps(cluster_data), headers=self.default_headers) self.tester.assertEquals(resp.status, 201) cluster = json.loads(resp.body) self.clusters.append(self.db.query(Cluster).get(cluster['id'])) else: cluster = Cluster() for field, value in cluster_data.iteritems(): setattr(cluster, field, value) self.db.add(cluster) self.db.commit() self.clusters.append(cluster) return cluster
def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ proxy_old_value = None if os.environ.get("http_proxy"): proxy_old_value = os.environ["http_proxy"] logger.warning("http_proxy variable is already set with " "value: {0}. Change to {1}. Old value " "will be restored after exit from script's " "execution context" .format(proxy_old_value, proxy)) os.environ["http_proxy"] = proxy try: yield except Exception as e: logger.exception("Error while talking to proxy. Details: {0}" .format(six.text_type(e))) finally: if proxy_old_value: logger.info("Restoring old value for http_proxy") os.environ["http_proxy"] = proxy_old_value else: logger.info("Deleting set http_proxy environment variable") del os.environ["http_proxy"]
def set_proxy(proxy): """Replace http_proxy environment variable for the scope of context execution. After exit from context old proxy value (if any) is restored :param proxy: - proxy url """ proxy_old_value = None if os.environ.get("http_proxy"): proxy_old_value = os.environ["http_proxy"] logger.warning("http_proxy variable is already set with " "value: {0}. Change to {1}. Old value " "will be restored after exit from script's " "execution context".format(proxy_old_value, proxy)) os.environ["http_proxy"] = proxy try: yield except Exception as e: logger.exception("Error while talking to proxy. Details: {0}".format( six.text_type(e))) finally: if proxy_old_value: logger.info("Restoring old value for http_proxy") os.environ["http_proxy"] = proxy_old_value else: logger.info("Deleting set http_proxy environment variable") del os.environ["http_proxy"]
def assign_network_to_interface(cls, instance, node): """Assign network to interface by default for single node Assign given network to first available interface. Checks interface type, if network is already assigned and already assigned networks. """ if instance is None: return from nailgun import objects untagged = cls.is_untagged(instance) dedicated = instance.meta.get('dedicated_nic') ifaces = objects.Node.get_interfaces_without_bonds_slaves(node) for iface in ifaces: if dedicated and iface.assigned_networks_list: continue for net in iface.assigned_networks_list: if net.meta.get('dedicated_nic'): break if net == instance: return if untagged and cls.is_untagged(net): break else: assigned_nets = iface.assigned_networks_list + [instance] objects.NIC.assign_networks(iface, assigned_nets) break else: logger.warning( "Cannot assign network %r appropriately for " "node %r. Set unassigned network to the " "interface %r", instance.name, node.name, ifaces[0].name) assigned_nets = ifaces[0].assigned_networks_list + [instance] objects.NIC.assign_networks(ifaces[0], assigned_nets)
def update_roles(cls, instance, new_roles): """Update roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_roles: list of new role names :returns: None """ if not instance.cluster_id: logger.warning( u"Attempting to assign roles to node " u"'{0}' which isn't added to cluster".format( instance.name or instance.id ) ) return if new_roles: instance.role_list = db().query(models.Role).filter_by( release_id=instance.cluster.release_id, ).filter( models.Role.name.in_(new_roles) ).all() else: instance.role_list = [] db().flush() db().refresh(instance)
def update_by_agent(cls, instance, data): """Update Node instance with some specific cases for agent. * don't update provisioning or error state back to discover * don't update volume information if disks arrays is empty :param data: dictionary of key-value pairs as object fields :returns: Node instance """ # don't update provisioning and error back to discover if instance.status in ('provisioning', 'error'): if data.get('status', 'discover') == 'discover': logger.debug( u"Node {0} has provisioning or error status - " u"status not updated by agent".format( instance.human_readable_name ) ) data['status'] = instance.status # don't update volume information, if agent has sent an empty array meta = data.get('meta', {}) if meta and len(meta.get('disks', [])) == 0 \ and instance.meta.get('disks'): logger.warning( u'Node {0} has received an empty disks array - ' u'volume information will not be updated'.format( instance.human_readable_name ) ) meta['disks'] = instance.meta['disks'] return cls.update(instance, data)
def update_volumes(cls, instance): """Update volumes for Node instance. Adds pending "disks" changes for Cluster which Node belongs to :param instance: Node instance :returns: None """ attrs = instance.attributes if not attrs: attrs = cls.create_attributes(instance) try: attrs.volumes = instance.volume_manager.gen_volumes_info() except Exception as exc: msg = (u"Failed to generate volumes " u"info for node '{0}': '{1}'").format( instance.name or instance.mac or instance.id, str(exc) or "see logs for details") logger.warning(traceback.format_exc()) Notification.create({ "topic": "error", "message": msg, "node_id": instance.id }) if instance.cluster_id: Cluster.add_pending_changes(instance.cluster, "disks", node_id=instance.id) db().add(attrs) db().flush()
def _update_dependencies(self): """Create dependencies that rely on regexp matching.""" for task in six.itervalues(self.node): # tasks and groups should be used for declaring dependencies # between tasks and roles (which are simply group of tasks) available_groups = self.get_groups_subgraph().nodes() for group in task.get('groups', ()): pattern = NameMatchingPolicy.create(group) not_matched = [] for available_group in available_groups: if pattern.match(available_group): self.add_edge(task['id'], available_group) else: not_matched.append(available_group) # Add dependency for non-existing group which will be # resolved in DeploymentGraphValidator if len(available_groups) == len(not_matched): self.add_edge(task['id'], group) logger.warning( 'Group "%s" is an invalid dependency', group) available_groups = not_matched for req in task.get('tasks', ()): self.add_edge(req, task['id'])
def update_volumes(cls, instance): attrs = instance.attributes if not attrs: attrs = cls.create_attributes(instance) try: attrs.volumes = instance.volume_manager.gen_volumes_info() except Exception as exc: msg = ( u"Failed to generate volumes " u"info for node '{0}': '{1}'" ).format( instance.name or instance.mac or instance.id, str(exc) or "see logs for details" ) logger.warning(traceback.format_exc()) Notification.create({ "topic": "error", "message": msg, "node_id": instance.id }) if instance.cluster_id: Cluster.add_pending_changes( instance.cluster, "disks", node_id=instance.id ) db().add(attrs) db().flush()
def _get_pxe_iface_name(cls, node): """Returns appropriate pxe iface's name In case when node has network scheme configured we can not rely on its pxe interface calculation algorithm anymore, because admin ip is moving to bridge and 'pxe' property will have 'False' value for all interfaces. In this case we should rely on db where actual pxe interface was saved during the bootstrap stage. In case when node for some reason has no pxe interface in db we should get pxe interface using appropriate function `get_admin_physical_iface`. """ db_interfaces = node.nic_interfaces pxe = next(( i for i in node.meta['interfaces'] if i.get('pxe') or i.get('mac') == node.mac), None) if pxe: return pxe.get('name') pxe_db = next((i for i in db_interfaces if i.pxe), None) if pxe_db: return pxe_db.name if db_interfaces: return objects.Node.get_admin_physical_iface(node).name logger.warning(u'Cannot find pxe interface for node "%s"', node.full_name) return None
def update_pending_roles(cls, instance, new_pending_roles): """Update pending_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_pending_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning( u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.full_name)) return logger.debug( u"Updating pending roles for node {0}: {1}".format( instance.full_name, new_pending_roles)) if new_pending_roles == []: # TODO(enchantner): research why the hell we need this Cluster.clear_pending_changes( instance.cluster, node_id=instance.id ) instance.pending_roles = new_pending_roles db().flush()
def stop_deployment_resp(cls, **kwargs): logger.info("RPC method stop_deployment_resp received: %s" % jsonutils.dumps(kwargs)) task_uuid = kwargs.get("task_uuid") nodes = kwargs.get("nodes", []) ia_nodes = kwargs.get("inaccessible_nodes", []) message = kwargs.get("error") status = kwargs.get("status") progress = kwargs.get("progress") task = objects.Task.get_by_uuid(task_uuid, fail_if_not_found=True) stopping_task_names = [consts.TASK_NAMES.deploy, consts.TASK_NAMES.deployment, consts.TASK_NAMES.provision] q_stop_tasks = objects.TaskCollection.filter_by_list(None, "name", stopping_task_names) q_stop_tasks = objects.TaskCollection.filter_by(q_stop_tasks, cluster_id=task.cluster_id) stop_tasks = objects.TaskCollection.order_by(q_stop_tasks, "id").all() # Locking cluster objects.Cluster.get_by_uid(task.cluster_id, fail_if_not_found=True, lock_for_update=True) if not stop_tasks: logger.warning( "stop_deployment_resp: deployment tasks \ not found for environment '%s'!", task.cluster_id, ) if status == consts.TASK_STATUSES.ready: task.cluster.status = consts.CLUSTER_STATUSES.stopped if stop_tasks: map(db().delete, stop_tasks) node_uids = [n["uid"] for n in itertools.chain(nodes, ia_nodes)] q_nodes = objects.NodeCollection.filter_by_id_list(None, node_uids) q_nodes = objects.NodeCollection.filter_by(q_nodes, cluster_id=task.cluster_id) q_nodes = objects.NodeCollection.order_by(q_nodes, "id") q_nodes = objects.NodeCollection.lock_for_update(q_nodes) # locking Nodes for update update_nodes = objects.NodeCollection.lock_for_update(q_nodes).all() for node in update_nodes: objects.Node.reset_to_discover(node) if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping") message = ( u"Deployment of environment '{0}' was successfully stopped. " u"Please make changes and reset the environment " u"if you want to redeploy it.".format(task.cluster.name or task.cluster_id) ) notifier.notify("done", message, task.cluster_id) data = {"status": status, "progress": progress, "message": message} objects.Task.update(task, data) cls._update_action_log_entry(status, task.name, task_uuid, nodes)
def update_primary_roles(cls, instance, new_primary_roles): """Update primary_roles for Node instance. Logs an error if node doesn't belong to Cluster :param instance: Node instance :param new_primary_roles: list of new pending role names :returns: None """ if not instance.cluster_id: logger.warning( u"Attempting to assign pending roles to node " u"'{0}' which isn't added to cluster".format( instance.full_name)) return assigned_roles = set(instance.roles + instance.pending_roles) for role in new_primary_roles: if role not in assigned_roles: logger.warning( u"Could not mark node {0} as primary for {1} role, " u"because there's no assigned {1} role.".format( instance.full_name, role) ) return logger.debug( u"Updating primary roles for node {0}: {1}".format( instance.full_name, new_primary_roles)) instance.primary_roles = new_primary_roles db().flush()
def get_nodes_by_role(cls, instance, role_name): """Get nodes related to some specific role :param instance: cluster db object :type: python object :param role_name: node role name :type: string """ role = db().query(models.Role).filter_by( release_id=instance.release_id, name=role_name).first() if not role: logger.warning("%s role doesn't exist", role_name) return [] nodes = db().query(models.Node).filter_by(cluster_id=instance.id) deployed_nodes = nodes.join( models.Node.role_list, aliased=True).filter( models.Role.id == role.id).all() pending_nodes = nodes.join( models.Node.pending_role_list, aliased=True).filter( models.Role.id == role.id).all() return deployed_nodes + pending_nodes
def POST(self): data = self.checked_data() node = Node() for key, value in data.iteritems(): if key == "meta": node.create_meta(value) else: setattr(node, key, value) node.name = "Untitled (%s)" % data['mac'][-5:] node.timestamp = datetime.now() self.db.add(node) self.db.commit() node.attributes = NodeAttributes() try: node.attributes.volumes = node.volume_manager.gen_volumes_info() if node.cluster: node.cluster.add_pending_changes( "disks", node_id=node.id ) except Exception as exc: msg = ( u"Failed to generate volumes " "info for node '{0}': '{1}'" ).format( node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details" ) logger.warning(traceback.format_exc()) notifier.notify("error", msg, node_id=node.id) self.db.add(node) self.db.commit() network_manager = NetworkManager() # Add interfaces for node from 'meta'. if node.meta and node.meta.get('interfaces'): network_manager.update_interfaces_info(node.id) if node.cluster_id: network_manager.allow_network_assignment_to_all_interfaces(node.id) network_manager.assign_networks_to_main_interface(node.id) try: ram = str(round(float( node.meta['memory']['total']) / 1073741824, 1)) except (KeyError, TypeError, ValueError): ram = "unknown" cores = str(node.meta.get('cpu', {}).get('total', "unknown")) notifier.notify("discover", "New node with %s CPU core(s) " "and %s GB memory is discovered" % (cores, ram), node_id=node.id) raise web.webapi.created(json.dumps( NodeHandler.render(node), indent=4 ))
def __init__(cls, name, bases, dct): super(HandlerRegistrator, cls).__init__(name, bases, dct) if hasattr(cls, 'model'): key = cls.model.__name__ if key in handlers: logger.warning("Handler for %s already registered" % key) return handlers[key] = cls
def __init__(cls, name, bases, dct): super(HandlerRegistrator, cls).__init__(name, bases, dct) if hasattr(cls, 'model'): key = cls.model.__name__ if key in handlers: logger.warning("Handler for %s already registered" % key) return handlers[key] = cls
def check_before_deployment(self, supertask): """Performs checks before deployment :param supertask: task SqlAlchemy object """ try: # if there are VIPs with same names in the network configuration # the error will be raised. Such situation may occur when, for # example, enabled plugins contain conflicting network # configuration network_info = self.serialize_network_cfg(self.cluster) except (errors.DuplicatedVIPNames, errors.NetworkRoleConflict) as e: raise errors.CheckBeforeDeploymentError(e.message) logger.info(u"Network info:\n{0}".format( jsonutils.dumps(network_info, indent=4))) # checking admin intersection with untagged network_info["networks"] = [ n for n in network_info["networks"] if n["name"] != "fuelweb_admin" ] check_networks = supertask.create_subtask( consts.TASK_NAMES.check_networks) self._call_silently(check_networks, tasks.CheckNetworksTask, data=network_info, check_all_parameters=True) if check_networks.status == consts.TASK_STATUSES.error: logger.warning("Checking networks failed: %s", check_networks.message) raise errors.CheckBeforeDeploymentError(check_networks.message) TaskHelper.set_ready_if_not_finished(check_networks) db().delete(check_networks) db().refresh(supertask) db().flush() # checking prerequisites check_before = supertask.create_subtask( consts.TASK_NAMES.check_before_deployment) logger.debug("Checking prerequisites task: %s", check_before.uuid) self._call_silently(check_before, tasks.CheckBeforeDeploymentTask) # if failed to check prerequisites # then task is already set to error if check_before.status == consts.TASK_STATUSES.error: logger.warning("Checking prerequisites failed: %s", check_before.message) raise errors.CheckBeforeDeploymentError(check_before.message) logger.debug( "Checking prerequisites is successful, starting deployment...") TaskHelper.set_ready_if_not_finished(check_before) db().delete(check_before) db().refresh(supertask) db().flush()
def stop_deployment_resp(cls, **kwargs): logger.info("RPC method stop_deployment_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) stop_tasks = db().query(Task).filter_by( cluster_id=task.cluster_id, ).filter( Task.name.in_(["deploy", "deployment", "provision"])).all() if not stop_tasks: logger.warning( "stop_deployment_resp: deployment tasks \ not found for environment '%s'!", task.cluster_id) if status == "ready": task.cluster.status = "stopped" if stop_tasks: map(db().delete, stop_tasks) db().commit() update_nodes = db().query(Node).filter( Node.id.in_( [n["uid"] for n in itertools.chain(nodes, ia_nodes)]), Node.cluster_id == task.cluster_id).yield_per(100) update_nodes.update( { "online": False, "status": "discover", "pending_addition": True }, synchronize_session='fetch') for n in update_nodes: n.roles, n.pending_roles = n.pending_roles, n.roles db().commit() if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping") message = (u"Deployment of environment '{0}' " u"was successfully stopped".format(task.cluster.name or task.cluster_id)) notifier.notify("done", message, task.cluster_id) TaskHelper.update_task_status(task_uuid, status, progress, message)
def PUT(self, node_id): """:returns: JSONized Node object. :http: * 200 (OK) * 400 (invalid node data specified) * 404 (node not found in db) """ node = self.get_object_or_404(Node, node_id) if not node.attributes: node.attributes = NodeAttributes(node_id=node.id) data = self.checked_data(self.validator.validate_update) network_manager = NetworkManager() old_cluster_id = node.cluster_id if data.get("pending_roles") == [] and node.cluster: node.cluster.clear_pending_changes(node_id=node.id) if "cluster_id" in data: if data["cluster_id"] is None and node.cluster: node.cluster.clear_pending_changes(node_id=node.id) node.roles = node.pending_roles = [] node.cluster_id = data["cluster_id"] if node.cluster_id != old_cluster_id: if old_cluster_id: network_manager.clear_assigned_networks(node) network_manager.clear_all_allowed_networks(node.id) if node.cluster_id: network_manager.assign_networks_by_default(node) network_manager.allow_network_assignment_to_all_interfaces(node) regenerate_volumes = any( ( "roles" in data and set(data["roles"]) != set(node.roles), "pending_roles" in data and set(data["pending_roles"]) != set(node.pending_roles), node.cluster_id != old_cluster_id, ) ) for key, value in data.iteritems(): # we don't allow to update id explicitly # and updated cluster_id before all other fields if key in ("id", "cluster_id"): continue setattr(node, key, value) if not node.status in ("provisioning", "deploying") and regenerate_volumes: try: node.attributes.volumes = node.volume_manager.gen_volumes_info() except Exception as exc: msg = (u"Failed to generate volumes " "info for node '{0}': '{1}'").format( node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details" ) logger.warning(traceback.format_exc()) notifier.notify("error", msg, node_id=node.id) db().commit() return self.render(node)
def POST(self): data = self.validator.validate(web.data()) node = Node() for key, value in data.iteritems(): setattr(node, key, value) node.name = "Untitled (%s)" % data['mac'][-5:] node.timestamp = datetime.now() self.db.add(node) self.db.commit() node.attributes = NodeAttributes() try: node.attributes.volumes = node.volume_manager.gen_volumes_info() if node.cluster: node.cluster.add_pending_changes( "disks", node_id=node.id ) except Exception as exc: msg = ( u"Failed to generate volumes " "info for node '{0}': '{1}'" ).format( node.name or data.get("mac") or data.get("id"), str(exc) or "see logs for details" ) logger.warning(traceback.format_exc()) notifier.notify("error", msg, node_id=node.id) self.db.add(node) self.db.commit() # Add interfaces for node from 'meta'. if node.meta and node.meta.get('interfaces'): nics = self.get_nics_from_meta(node) map(self.db.add, nics) self.db.commit() if node.cluster_id: self.allow_network_assignment_to_all_interfaces(node) self.assign_networks_to_main_interface(node) self.db.commit() try: ram = str(round(float( node.meta['memory']['total']) / 1073741824, 1)) except (KeyError, TypeError, ValueError): ram = "unknown" cores = str(node.meta.get('cpu', {}).get('total', "unknown")) notifier.notify("discover", "New node with %s CPU core(s) " "and %s GB memory is discovered" % (cores, ram), node_id=node.id) raise web.webapi.created(json.dumps( NodeHandler.render(node), indent=4 ))
def set_default_attributes(cls, instance): if not instance.cluster_id: logger.warning( u"Attempting to update attributes of node " u"'{0}' which isn't added to any cluster".format( instance.full_name)) return instance.attributes = instance.cluster.release.node_attributes
def pending_roles(self, new_roles): if not self.cluster: logger.warning(u"Attempting to assign pending_roles to node " u"'{0}' which isn't added to cluster".format( self.name or self.id)) return self.pending_role_list = db().query(Role).filter_by( release_id=self.cluster.release_id, ).filter( Role.name.in_(new_roles)).all()
def _update_nodes(transaction, nodes_instances, nodes_params): allow_update = { 'name', 'status', 'hostname', 'kernel_params', 'pending_addition', 'pending_deletion', 'error_msg', 'online', 'progress', } # dry-run transactions must not update nodes except progress column if transaction.dry_run: allow_update = {'progress'} for node in nodes_instances: node_params = nodes_params.pop(node.uid) for param in allow_update.intersection(node_params): if param == 'status': new_status = node_params['status'] if new_status == 'deleted': # the deleted is special status which causes # to delete node from cluster objects.Node.remove_from_cluster(node) elif new_status == 'error': # TODO(bgaifullin) do not persist status in DB node.status = new_status node.error_type = node_params.get( 'error_type', consts.NODE_ERRORS.deploy) node.progress = 100 # Notification on particular node failure notifier.notify(consts.NOTIFICATION_TOPICS.error, u"Node '{0}' failed: {1}".format( node.name, node_params.get( 'error_msg', "Unknown error")), cluster_id=transaction.cluster_id, node_id=node.uid, task_uuid=transaction.uuid) elif new_status == 'ready': # TODO(bgaifullin) need to remove pengind roles concept node.roles = list(set(node.roles + node.pending_roles)) node.pending_roles = [] node.progress = 100 node.status = new_status else: node.status = new_status else: setattr(node, param, node_params[param]) db.flush() if nodes_params: logger.warning("The following nodes are not found: %s", ",".join(sorted(nodes_params.keys())))
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ from nailgun.objects import Release plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] propagate_task_deploy = get_in( attributes, 'common', 'propagate_task_deploy', 'value') if propagate_task_deploy is not None: legacy_tasks_are_ignored = not propagate_task_deploy else: legacy_tasks_are_ignored = not get_in( cluster.attributes.editable, 'common', 'propagate_task_deploy', 'value') for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning( 'Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled'] \ and plugin_id == container['chosen_id'] if (enabled and Release.is_lcm_supported(cluster.release) and legacy_tasks_are_ignored and cls.contains_legacy_tasks( wrap_plugin(Plugin.get_by_uid(plugin.id)))): raise errors.InvalidData( 'Cannot enable plugin with legacy tasks unless ' 'propagate_task_deploy attribute is set. ' 'Ensure tasks.yaml is empty and all tasks ' 'has version >= 2.0.0.') ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None )
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ from nailgun.objects import Release plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] propagate_task_deploy = get_in(attributes, 'common', 'propagate_task_deploy', 'value') if propagate_task_deploy is not None: legacy_tasks_are_ignored = not propagate_task_deploy else: legacy_tasks_are_ignored = not get_in( cluster.attributes.editable, 'common', 'propagate_task_deploy', 'value') for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled'] \ and plugin_id == container['chosen_id'] if (enabled and Release.is_lcm_supported(cluster.release) and legacy_tasks_are_ignored and cls.contains_legacy_tasks( wrap_plugin(Plugin.get_by_uid(plugin.id)))): raise errors.InvalidData( 'Cannot enable plugin with legacy tasks unless ' 'propagate_task_deploy attribute is set. ' 'Ensure tasks.yaml is empty and all tasks ' 'has version >= 2.0.0.') ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None)
def set_primary_role(cls, intance, nodes, role_name): """Method for assigning primary attribute for specific role. - verify that there is no primary attribute of specific role assigned to cluster nodes with this role in role list or pending role list, and this node is not marked for deletion - if there is no primary role assigned, filter nodes which have current role in roles_list or pending_role_list - if there is nodes with ready state - they should have higher priority - if role was in primary_role_list - change primary attribute for that association, same for role_list, this is required because deployment_serializer used by cli to generate deployment info :param instance: Cluster db objects :param nodes: list of Node db objects :param role_name: string with known role name """ all_roles = intance.release.role_list role = next((r for r in all_roles if r.name == role_name), None) if role is None: logger.warning('Trying to assign primary for non-existing role %s', role_name) return node = db().query( models.Node).filter_by(pending_deletion=False).filter( or_( models.Node.role_associations.any(role=role.id, primary=True), models.Node.pending_role_associations.any( role=role.id, primary=True))).filter( models.Node.cluster == intance).first() if not node: filtered_nodes = [] for node in nodes: if (not node.pending_deletion and (role in node.role_list or role in node.pending_role_list)): filtered_nodes.append(node) filtered_nodes = sorted(filtered_nodes, key=lambda node: node.id) if filtered_nodes: primary_node = next( (node for node in filtered_nodes if node.status == consts.NODE_STATUSES.ready), filtered_nodes[0]) if role in primary_node.role_list: associations = primary_node.role_associations elif role in primary_node.pending_role_list: associations = primary_node.pending_role_associations else: logger.warning(('Role %s neither in pending, nor role.' ' You hit strange bug'), role_name) return for assoc in associations: if assoc.role == role.id: assoc.primary = True db().flush()
def update_by_agent(cls, instance, data): """Update Node instance with some specific cases for agent. * don't update provisioning or error state back to discover * don't update volume information if disks arrays is empty :param data: dictionary of key-value pairs as object fields :returns: Node instance """ # don't update provisioning and error back to discover data_status = data.get('status') if instance.status in ('provisioning', 'error'): if data.get('status', 'discover') == 'discover': logger.debug(u"Node {0} has provisioning or error status - " u"status not updated by agent".format( instance.human_readable_name)) data.pop('status', None) meta = data.get('meta', {}) # don't update volume information, if agent has sent an empty array if len(meta.get('disks', [])) == 0 and instance.meta.get('disks'): logger.warning(u'Node {0} has received an empty disks array - ' u'volume information will not be updated'.format( instance.human_readable_name)) meta['disks'] = instance.meta['disks'] # don't update volume information, if it is locked by node status if 'disks' in meta and cls.hardware_info_locked(instance): logger.debug("Volume information is locked for update on node %s", instance.human_readable_name) meta['disks'] = instance.meta['disks'] # (dshulyak) change this verification to NODE_STATUSES.deploying # after we will reuse ips from dhcp range if data.get('ip'): if instance.cluster_id: update_status = cls.check_ip_belongs_to_own_admin_network( instance, data['ip']) else: update_status = cls.check_ip_belongs_to_any_admin_network( instance, data['ip']) if update_status: if instance.status == consts.NODE_STATUSES.error and \ instance.error_type == consts.NODE_ERRORS.discover: # accept the status from agent if the node had wrong IP # previously if data_status: instance.status = data_status else: instance.status = consts.NODE_STATUSES.discover else: data.pop('status', None) return cls.update(instance, data)