def remove_cluster_resp(cls, **kwargs): logger.info("RPC method remove_cluster_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') cls.remove_nodes_resp(**kwargs) task = TaskHelper.get_task_by_uuid(task_uuid) cluster = task.cluster if task.status in ('ready', ): logger.debug("Removing environment itself") cluster_name = cluster.name ips = db().query(IPAddr).filter( IPAddr.network.in_([n.id for n in cluster.network_groups])) map(db().delete, ips) db().flush() db().delete(cluster) db().flush() notifier.notify( "done", u"Environment '%s' and all its nodes are deleted" % (cluster_name)) elif task.status in ('error', ): cluster.status = 'error' db().add(cluster) db().flush() if not task.message: task.message = "Failed to delete nodes:\n{0}".format( cls._generate_error_message(task, error_types=('deletion', ))) notifier.notify("error", task.message, cluster.id)
def check_redhat_credentials_resp(cls, **kwargs): logger.info("RPC method check_redhat_credentials_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) release_info = task.cache['args']['release_info'] release_id = release_info['release_id'] release = db().query(Release).get(release_id) if not release: logger.error( "download_release_resp: Release" " with ID %s not found", release_id) return if error_msg: status = 'error' cls._update_release_state(release_id, 'error') # TODO(NAME): remove this ugly checks if 'Unknown error' in error_msg: error_msg = 'Failed to check Red Hat ' \ 'credentials' if error_msg != 'Task aborted': notifier.notify('error', error_msg) result = {"release_info": {"release_id": release_id}} TaskHelper.update_task_status(task_uuid, status, progress, error_msg, result)
def download_release_resp(cls, **kwargs): logger.info("RPC method download_release_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) release_info = task.cache['args']['release_info'] release_id = release_info['release_id'] release = db().query(Release).get(release_id) if not release: logger.error( "download_release_resp: Release" " with ID %s not found", release_id) return if error_msg: status = 'error' error_msg = "{0} download and preparation " \ "has failed.".format(release.name) cls._download_release_error(release_id, error_msg) elif progress == 100 and status == 'ready': cls._download_release_completed(release_id) result = {"release_info": {"release_id": release_id}} TaskHelper.update_task_status(task_uuid, status, progress, error_msg, result)
def stop_deployment_resp(cls, **kwargs): logger.info("RPC method stop_deployment_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) stop_tasks = db().query(Task).filter_by( cluster_id=task.cluster_id, ).filter( Task.name.in_(["deploy", "deployment", "provision"])).all() if not stop_tasks: logger.warning( "stop_deployment_resp: deployment tasks \ not found for environment '%s'!", task.cluster_id) if status == "ready": task.cluster.status = "stopped" if stop_tasks: map(db().delete, stop_tasks) db().commit() update_nodes = db().query(Node).filter( Node.id.in_( [n["uid"] for n in itertools.chain(nodes, ia_nodes)]), Node.cluster_id == task.cluster_id).yield_per(100) update_nodes.update( { "online": False, "status": "discover", "pending_addition": True }, synchronize_session='fetch') for n in update_nodes: n.roles, n.pending_roles = n.pending_roles, n.roles db().commit() if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping") message = (u"Deployment of environment '{0}' " u"was successfully stopped".format(task.cluster.name or task.cluster_id)) notifier.notify("done", message, task.cluster_id) TaskHelper.update_task_status(task_uuid, status, progress, message)
def reset_environment_resp(cls, **kwargs): logger.info("RPC method reset_environment_resp received: %s", json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) if status == "ready": # restoring pending changes task.cluster.status = "new" objects.Cluster.add_pending_changes(task.cluster, "attributes") objects.Cluster.add_pending_changes(task.cluster, "networks") for node in task.cluster.nodes: objects.Cluster.add_pending_changes(task.cluster, "disks", node_id=node.id) update_nodes = db().query(Node).filter( Node.id.in_( [n["uid"] for n in itertools.chain(nodes, ia_nodes)]), Node.cluster_id == task.cluster_id).yield_per(100) update_nodes.update( { "online": False, "status": "discover", "pending_addition": True, "pending_deletion": False, }, synchronize_session='fetch') for n in update_nodes: n.roles, n.pending_roles = n.pending_roles, n.roles db().commit() if ia_nodes: cls._notify_inaccessible(task.cluster_id, [n["uid"] for n in ia_nodes], u"environment resetting") message = (u"Environment '{0}' " u"was successfully reset".format(task.cluster.name or task.cluster_id)) notifier.notify("done", message, task.cluster_id) TaskHelper.update_task_status(task.uuid, status, progress, message)
def provision_resp(cls, **kwargs): logger.info( "RPC method provision_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') nodes = kwargs.get('nodes', []) task = TaskHelper.get_task_by_uuid(task_uuid) for node in nodes: uid = node.get('uid') node_db = db().query(Node).get(uid) if not node_db: logger.warn('Node with uid "{0}" not found'.format(uid)) continue if node.get('status') == 'error': node_db.status = 'error' node_db.progress = 100 node_db.error_type = 'provision' node_db.error_msg = node.get('error_msg', 'Unknown error') else: node_db.status = node.get('status') node_db.progress = node.get('progress') db().commit() task = TaskHelper.get_task_by_uuid(task_uuid) if nodes and not progress: progress = TaskHelper.recalculate_provisioning_task_progress(task) TaskHelper.update_task_status(task.uuid, status, progress, message)
def provision_resp(cls, **kwargs): logger.info("RPC method provision_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') nodes = kwargs.get('nodes', []) task = TaskHelper.get_task_by_uuid(task_uuid) for node in nodes: uid = node.get('uid') node_db = db().query(Node).get(uid) if not node_db: logger.warn('Node with uid "{0}" not found'.format(uid)) continue if node.get('status') == 'error': node_db.status = 'error' node_db.progress = 100 node_db.error_type = 'provision' node_db.error_msg = node.get('error_msg', 'Unknown error') else: node_db.status = node.get('status') node_db.progress = node.get('progress') db().commit() task = TaskHelper.get_task_by_uuid(task_uuid) if nodes and not progress: progress = TaskHelper.recalculate_provisioning_task_progress(task) TaskHelper.update_task_status(task.uuid, status, progress, message)
def remove_cluster_resp(cls, **kwargs): logger.info( "RPC method remove_cluster_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') cls.remove_nodes_resp(**kwargs) task = TaskHelper.get_task_by_uuid(task_uuid) cluster = task.cluster if task.status in ('ready',): logger.debug("Removing environment itself") cluster_name = cluster.name ips = db().query(IPAddr).filter( IPAddr.network.in_([n.id for n in cluster.network_groups]) ) map(db().delete, ips) db().flush() db().delete(cluster) db().flush() notifier.notify( "done", u"Environment '%s' and all its nodes are deleted" % ( cluster_name ) ) elif task.status in ('error',): cluster.status = 'error' db().add(cluster) db().flush() if not task.message: task.message = "Failed to delete nodes:\n{0}".format( cls._generate_error_message( task, error_types=('deletion',) ) ) notifier.notify( "error", task.message, cluster.id )
def redhat_check_licenses_resp(cls, **kwargs): logger.info( "RPC method redhat_check_licenses_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') notify = kwargs.get('msg') task = TaskHelper.get_task_by_uuid(task_uuid) release_info = task.cache['args']['release_info'] release_id = release_info['release_id'] release = db().query(Release).get(release_id) if not release: logger.error("download_release_resp: Release" " with ID %s not found", release_id) return if error_msg: status = 'error' cls._update_release_state(release_id, 'error') # TODO(NAME): remove this ugly checks if 'Unknown error' in error_msg: error_msg = 'Failed to check Red Hat licenses ' if error_msg != 'Task aborted': notifier.notify('error', error_msg) if notify: notifier.notify('error', notify) result = { "release_info": { "release_id": release_id } } TaskHelper.update_task_status( task_uuid, status, progress, error_msg, result )
def download_release_resp(cls, **kwargs): logger.info( "RPC method download_release_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) release_info = task.cache['args']['release_info'] release_id = release_info['release_id'] release = db().query(Release).get(release_id) if not release: logger.error("download_release_resp: Release" " with ID %s not found", release_id) return if error_msg: status = 'error' error_msg = "{0} download and preparation " \ "has failed.".format(release.name) cls._download_release_error( release_id, error_msg ) elif progress == 100 and status == 'ready': cls._download_release_completed(release_id) result = { "release_info": { "release_id": release_id } } TaskHelper.update_task_status( task_uuid, status, progress, error_msg, result )
def verify_networks_resp(cls, **kwargs): logger.info( "RPC method verify_networks_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') # We simply check that each node received all vlans for cluster task = TaskHelper.get_task_by_uuid(task_uuid) result = [] # We expect that 'nodes' contains all nodes which we test. # Situation when some nodes not answered must be processed # in orchestrator early. if nodes is None: # If no nodes in kwargs then we update progress or status only. pass elif isinstance(nodes, list): cached_nodes = task.cache['args']['nodes'] node_uids = [str(n['uid']) for n in nodes] cached_node_uids = [str(n['uid']) for n in cached_nodes] forgotten_uids = set(cached_node_uids) - set(node_uids) if forgotten_uids: absent_nodes = db().query(Node).filter( Node.id.in_(forgotten_uids) ).all() absent_node_names = [] for n in absent_nodes: if n.name: absent_node_names.append(n.name) else: absent_node_names.append('id: %s' % n.id) if not error_msg: error_msg = 'Node(s) {0} didn\'t return data.'.format( ', '.join(absent_node_names) ) status = 'error' else: error_nodes = [] for node in nodes: cached_nodes_filtered = filter( lambda n: str(n['uid']) == str(node['uid']), cached_nodes ) if not cached_nodes_filtered: logger.warning( "verify_networks_resp: arguments contain node " "data which is not in the task cache: %r", node ) continue cached_node = cached_nodes_filtered[0] for cached_network in cached_node['networks']: received_networks_filtered = filter( lambda n: n['iface'] == cached_network['iface'], node.get('networks', []) ) if received_networks_filtered: received_network = received_networks_filtered[0] absent_vlans = list( set(cached_network['vlans']) - set(received_network['vlans']) ) else: logger.warning( "verify_networks_resp: arguments don't contain" " data for interface: uid=%s iface=%s", node['uid'], cached_network['iface'] ) absent_vlans = cached_network['vlans'] if absent_vlans: data = {'uid': node['uid'], 'interface': cached_network['iface'], 'absent_vlans': absent_vlans} node_db = db().query(Node).get(node['uid']) if node_db: data['name'] = node_db.name db_nics = filter( lambda i: i.name == cached_network['iface'], node_db.interfaces ) if db_nics: data['mac'] = db_nics[0].mac else: logger.warning( "verify_networks_resp: can't find " "interface %r for node %r in DB", cached_network['iface'], node_db.id ) data['mac'] = 'unknown' else: logger.warning( "verify_networks_resp: can't find node " "%r in DB", node['uid'] ) error_nodes.append(data) if error_nodes: result = error_nodes status = 'error' else: error_msg = (error_msg or 'verify_networks_resp: argument "nodes"' ' have incorrect type') status = 'error' logger.error(error_msg) if status not in ('ready', 'error'): TaskHelper.update_task_status(task_uuid, status, progress, error_msg, result) else: TaskHelper.update_verify_networks(task_uuid, status, progress, error_msg, result)
def reset_environment_resp(cls, **kwargs): logger.info( "RPC method reset_environment_resp received: %s", json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) if status == "ready": # restoring pending changes task.cluster.status = "new" objects.Cluster.add_pending_changes(task.cluster, "attributes") objects.Cluster.add_pending_changes(task.cluster, "networks") for node in task.cluster.nodes: objects.Cluster.add_pending_changes( task.cluster, "disks", node_id=node.id ) update_nodes = db().query(Node).filter( Node.id.in_([ n["uid"] for n in itertools.chain( nodes, ia_nodes ) ]), Node.cluster_id == task.cluster_id ).yield_per(100) update_nodes.update( { "online": False, "status": "discover", "pending_addition": True, "pending_deletion": False, }, synchronize_session='fetch' ) for n in update_nodes: n.roles, n.pending_roles = n.pending_roles, n.roles db().commit() if ia_nodes: cls._notify_inaccessible( task.cluster_id, [n["uid"] for n in ia_nodes], u"environment resetting" ) message = ( u"Environment '{0}' " u"was successfully reset".format( task.cluster.name or task.cluster_id ) ) notifier.notify( "done", message, task.cluster_id ) TaskHelper.update_task_status( task.uuid, status, progress, message )
def stop_deployment_resp(cls, **kwargs): logger.info( "RPC method stop_deployment_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes', []) ia_nodes = kwargs.get('inaccessible_nodes', []) message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) stop_tasks = db().query(Task).filter_by( cluster_id=task.cluster_id, ).filter( Task.name.in_(["deploy", "deployment", "provision"]) ).all() if not stop_tasks: logger.warning("stop_deployment_resp: deployment tasks \ not found for environment '%s'!", task.cluster_id) if status == "ready": task.cluster.status = "stopped" if stop_tasks: map(db().delete, stop_tasks) db().commit() update_nodes = db().query(Node).filter( Node.id.in_([ n["uid"] for n in itertools.chain( nodes, ia_nodes ) ]), Node.cluster_id == task.cluster_id ).yield_per(100) update_nodes.update( { "online": False, "status": "discover", "pending_addition": True }, synchronize_session='fetch' ) for n in update_nodes: n.roles, n.pending_roles = n.pending_roles, n.roles db().commit() if ia_nodes: cls._notify_inaccessible( task.cluster_id, [n["uid"] for n in ia_nodes], u"deployment stopping" ) message = ( u"Deployment of environment '{0}' " u"was successfully stopped".format( task.cluster.name or task.cluster_id ) ) notifier.notify( "done", message, task.cluster_id ) TaskHelper.update_task_status( task_uuid, status, progress, message )
def test_get_task_by_uuid_returns_task(self): task = Task(name='deploy') self.db.add(task) self.db.commit() task_by_uuid = TaskHelper.get_task_by_uuid(task.uuid) self.assertEquals(task.uuid, task_by_uuid.uuid)
def verify_networks_resp(cls, **kwargs): logger.info("RPC method verify_networks_resp received: %s" % json.dumps(kwargs)) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') error_msg = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') # We simply check that each node received all vlans for cluster task = TaskHelper.get_task_by_uuid(task_uuid) result = [] # We expect that 'nodes' contains all nodes which we test. # Situation when some nodes not answered must be processed # in orchestrator early. if nodes is None: # If no nodes in kwargs then we update progress or status only. pass elif isinstance(nodes, list): cached_nodes = task.cache['args']['nodes'] node_uids = [str(n['uid']) for n in nodes] cached_node_uids = [str(n['uid']) for n in cached_nodes] forgotten_uids = set(cached_node_uids) - set(node_uids) if forgotten_uids: absent_nodes = db().query(Node).filter( Node.id.in_(forgotten_uids)).all() absent_node_names = [] for n in absent_nodes: if n.name: absent_node_names.append(n.name) else: absent_node_names.append('id: %s' % n.id) if not error_msg: error_msg = 'Node(s) {0} didn\'t return data.'.format( ', '.join(absent_node_names)) status = 'error' else: error_nodes = [] for node in nodes: cached_nodes_filtered = filter( lambda n: str(n['uid']) == str(node['uid']), cached_nodes) if not cached_nodes_filtered: logger.warning( "verify_networks_resp: arguments contain node " "data which is not in the task cache: %r", node) continue cached_node = cached_nodes_filtered[0] for cached_network in cached_node['networks']: received_networks_filtered = filter( lambda n: n['iface'] == cached_network['iface'], node.get('networks', [])) if received_networks_filtered: received_network = received_networks_filtered[0] absent_vlans = list( set(cached_network['vlans']) - set(received_network['vlans'])) else: logger.warning( "verify_networks_resp: arguments don't contain" " data for interface: uid=%s iface=%s", node['uid'], cached_network['iface']) absent_vlans = cached_network['vlans'] if absent_vlans: data = { 'uid': node['uid'], 'interface': cached_network['iface'], 'absent_vlans': absent_vlans } node_db = db().query(Node).get(node['uid']) if node_db: data['name'] = node_db.name db_nics = filter( lambda i: i.name == cached_network[ 'iface'], node_db.interfaces) if db_nics: data['mac'] = db_nics[0].mac else: logger.warning( "verify_networks_resp: can't find " "interface %r for node %r in DB", cached_network['iface'], node_db.id) data['mac'] = 'unknown' else: logger.warning( "verify_networks_resp: can't find node " "%r in DB", node['uid']) error_nodes.append(data) if error_nodes: result = error_nodes status = 'error' else: error_msg = (error_msg or 'verify_networks_resp: argument "nodes"' ' have incorrect type') status = 'error' logger.error(error_msg) if status not in ('ready', 'error'): TaskHelper.update_task_status(task_uuid, status, progress, error_msg, result) else: TaskHelper.update_verify_networks(task_uuid, status, progress, error_msg, result)
def deploy_resp(cls, **kwargs): logger.info( "RPC method deploy_resp received: %s" % json.dumps(kwargs) ) task_uuid = kwargs.get('task_uuid') nodes = kwargs.get('nodes') or [] message = kwargs.get('error') status = kwargs.get('status') progress = kwargs.get('progress') task = TaskHelper.get_task_by_uuid(task_uuid) if not task: # No task found - nothing to do here, returning logger.warning( u"No task with uuid '{0}'' found - nothing changed".format( task_uuid ) ) return if not status: status = task.status # First of all, let's update nodes in database for node in nodes: node_db = db().query(Node).get(node['uid']) if not node_db: logger.warning( u"No node found with uid '{0}' - nothing changed".format( node['uid'] ) ) continue update_fields = ( 'error_msg', 'error_type', 'status', 'progress', 'online' ) for param in update_fields: if param in node: logger.debug( u"Updating node {0} - set {1} to {2}".format( node['uid'], param, node[param] ) ) setattr(node_db, param, node[param]) if param == 'progress' and node.get('status') == 'error' \ or node.get('online') is False: # If failure occurred with node # it's progress should be 100 node_db.progress = 100 # Setting node error_msg for offline nodes if node.get('online') is False \ and not node_db.error_msg: node_db.error_msg = u"Node is offline" # Notification on particular node failure notifier.notify( "error", u"Failed to deploy node '{0}': {1}".format( node_db.name, node_db.error_msg or "Unknown error" ), cluster_id=task.cluster_id, node_id=node['uid'], task_uuid=task_uuid ) db().add(node_db) db().commit() # We should calculate task progress by nodes info task = TaskHelper.get_task_by_uuid(task_uuid) if nodes and not progress: progress = TaskHelper.recalculate_deployment_task_progress(task) # Let's check the whole task status if status in ('error',): cls._error_action(task, status, progress, message) elif status in ('ready',): cls._success_action(task, status, progress) else: TaskHelper.update_task_status(task.uuid, status, progress, message)