Example #1
0
    def _get_specific_core_para(rcp_msg, args):
        try:
            interface_local = None
            ccap_core_ip = None
            if rcp_msg.RcpMessageType in [
                    t_RcpMessage.CONNECT_CLOSE_NOTIFICATION,
                    t_RcpMessage.SESSION_INITIATED,
                    t_RcpMessage.REDIRECT_NOTIFICATION
            ]:
                core_para = json.loads(rcp_msg.parameter)
                ccap_core_ip = core_para[
                    'addr_remote'] if None is not core_para[
                        'addr_remote'] else ''
                interface_local = core_para[
                    'interface_local'] if None is not core_para[
                        'interface_local'] else ''
            elif rcp_msg.RcpMessageType in [
                    t_RcpMessage.RPD_CONFIGURATION,
                    t_RcpMessage.RPD_CONFIGURATION_DONE,
                    t_RcpMessage.IRA_RECEIVED, t_RcpMessage.REX_RECEIVED
            ]:
                if args:
                    session, transaction_identifier, trans_id = args
                    interface_local = session.get_descriptor().interface_local
                    ccap_core_ip = Convert.format_ip(
                        session.get_descriptor().addr_remote)
            specific = (interface_local, ccap_core_ip)
            return specific

        except Exception as e:
            return None
Example #2
0
 def handle_init_conf_completed(self, args):
     status_changed = False
     specific = None
     if None is not args:
         session, transaction_identifier, trans_id = args
         interface_local = session.get_descriptor().interface_local
         ccap_core_ip = Convert.format_ip(
             session.get_descriptor().addr_remote)
         if (interface_local, ccap_core_ip) in self.rcp:
             if self.rcp[(interface_local,
                          ccap_core_ip)]['status'] != self.UP:
                 status_changed = True
                 specific = (interface_local, ccap_core_ip)
                 self.rcp[(interface_local,
                           ccap_core_ip)]['status'] = self.UP
                 if interface_local == self.principal_core_interface and ccap_core_ip == self.principal_core:
                     self.process.orchestrator.set_active_principal_core(
                         interface_local, ccap_core_ip)
     return status_changed, specific
Example #3
0
    def rcp_msg_cb(self, seq, args=None):
        """Send 'RpdCapabilities', 'CcapCoreIdentification' to manager.

        :param seq: data format t_RcpMessage defined in rcp.proto or
         rcp sequence
        :param args: RCP message arguments

        """
        status_changed = False
        specific = None
        core_msg_str = None
        if None is seq:
            self.logger.error("Parameters error, can not be NoneType")
            return
        elif isinstance(seq, t_RcpMessage):
            rcp_msg = seq
        else:
            rcp_msg = seq.ipc_msg
        self.logger.info("rcp_msg_cb: %s args, %s", str(rcp_msg), str(args))
        interface_local = ''
        if args:
            session, transaction_identifier, trans_id = args
            interface_local = session.get_descriptor().interface_local
        self.logger.info("RCP message type: %s",
                         rcp_msg.t_RcpMessageType.Name(rcp_msg.RcpMessageType))

        if rcp_msg.RcpMessageType == t_RcpMessage.RPD_REBOOT:
            self.logger.debug("Received RPD Reboot from RCP")
            core_ip = ''
            info = ''
            if rcp_msg.HasField('parameter'):
                core_para = json.loads(rcp_msg.parameter)
                core_ip = core_para['addr_remote'] if None is not core_para[
                    'addr_remote'] else ''
                interface_local = core_para[
                    'interface_local'] if None is not core_para[
                        'interface_local'] else ''
                info = core_para['info'] if None is not core_para[
                    'info'] else ''
            # FixMe, may need to send reboot msg to mgr
            for idx in self.mgrs:
                event_request_rsp = protoDef.msg_event_notification()
                event_request_rsp.mgr_event.mgr_id = idx
                event_request_rsp.mgr_event.event_id = self.id
                event_request_rsp.mgr_event.data = json.dumps("reboot/" +
                                                              interface_local +
                                                              ';' + core_ip +
                                                              ';' + info)
                self.mgrs[idx]['transport'].sock.send(
                    event_request_rsp.SerializeToString(), flags=zmq.NOBLOCK)
                self.logger.debug("Send event notification to id %s, msg:%s" %
                                  (idx, event_request_rsp))

        elif rcp_msg.RcpMessageType == t_RcpMessage.REDIRECT_NOTIFICATION:
            self.logger.debug("Received RPD Redirect message from RCP")
            #  need to send message to manager to handle this
            core_ip = ''
            if rcp_msg.HasField('parameter'):
                core_para = json.loads(rcp_msg.parameter)
                core_ip = core_para['addr_remote'] if None is not core_para[
                    'addr_remote'] else ''
                interface_local = core_para[
                    'interface_local'] if None is not core_para[
                        'interface_local'] else ''
            for idx in self.mgrs:
                event_request_rsp = protoDef.msg_event_notification()
                event_request_rsp.mgr_event.mgr_id = idx
                event_request_rsp.mgr_event.event_id = self.id
                event_request_rsp.mgr_event.data = json.dumps(
                    "redirect/" + ";".join([
                        Convert.format_ip(core)
                        for core in rcp_msg.RedirectCCAPAddresses
                    ]) + '/' + interface_local + ';' + core_ip)
                self.mgrs[idx]['transport'].sock.send(
                    event_request_rsp.SerializeToString(), flags=zmq.NOBLOCK)
                self.logger.debug("Send manager event to id %s, msg:%s" %
                                  (idx, event_request_rsp))

            # changed the status, then send to corresponding requester
            if interface_local != '' and core_ip != '':
                self.logger.info("The redirected core is: (%s, %s)",
                                 interface_local, core_ip)
                if (interface_local, core_ip) in self.rcp:
                    if self.rcp[(interface_local,
                                 core_ip)]['status'] != self.DOWN:
                        status_changed = True
                        specific = (interface_local, core_ip)
                        self.rcp[(interface_local,
                                  core_ip)]['status'] = self.DOWN

        elif rcp_msg.RcpMessageType == t_RcpMessage.RPD_CONFIGURATION:
            self.logger.debug(
                "Received RPD Ccap Core configuration message from RCP")
            cfg_data = rcp_msg.RpdDataMessage.RpdData
            if rcp_msg.HasField('parameter'):
                interface_local = rcp_msg.parameter

            for descr, value in cfg_data.ListFields():
                if descr.name == 'CcapCoreIdentification':
                    #  need to send message to manager to handle this
                    for cap_info in value:
                        coreIpAddress = "0.0.0.0"
                        identRecord = CcapCoreIdentification()
                        index = -1
                        is_principal = False
                        if cap_info.HasField("Index"):
                            index = cap_info.Index
                            identRecord.index = index
                            identRecord.read()
                            coreIpAddress = identRecord.core_ip_addr
                            is_principal = identRecord.is_principal
                            is_active = identRecord.is_active
                            if cap_info.HasField("CoreIpAddress"):
                                coreIpAddress = cap_info.CoreIpAddress
                            if cap_info.HasField("IsPrincipal"):
                                is_principal = cap_info.IsPrincipal
                            if cap_info.HasField("CoreMode"):
                                is_active = cap_info.CoreMode is t_CcapCoreIdentification.COREMODEACTIVE \
                                    if cap_info.HasField("CoreMode") else True
                            self.logger.debug(
                                "CcapCoreIdentification index =%d coreIpAddress is %s is_principal=%d is_active=%d",
                                index, coreIpAddress, is_principal, is_active)
                        else:
                            continue
                        caps = {
                            "index": index,
                            "is_active": is_active,
                            "ccap_core": Convert.format_ip(coreIpAddress),
                            "interface": interface_local,
                            "is_principal": is_principal
                        }
                        for idx in self.mgrs:
                            event_request_rsp = protoDef.msg_event_notification(
                            )
                            event_request_rsp.mgr_event.mgr_id = idx
                            event_request_rsp.mgr_event.event_id = self.id
                            event_request_rsp.mgr_event.data = json.dumps(
                                "role/" + json.dumps(caps))
                            self.mgrs[idx]['transport'].sock.send(
                                event_request_rsp.SerializeToString(),
                                flags=zmq.NOBLOCK)
                            self.logger.debug(
                                "Send event notification to id %s, msg:%s" %
                                (idx, event_request_rsp))

                        # changed the status, then send to corresponding requester, just for CLI test
                        for interface, core_ip in self.rcp:
                            if interface != interface_local or \
                                    not Convert.is_ip_address_equal(core_ip, coreIpAddress):
                                continue
                            ccap_core = self.rcp[(interface, core_ip)]
                            # send the Core Identification
                            for core_id in ccap_core['requester']:
                                info_update = protoDef.msg_event_notification()
                                info_update.agent_info_update.ccap_core_id = core_id

                                # The ugly code is caused by the proto file is different place
                                if cap_info.HasField("CoreId"):
                                    info_update.agent_info_update.ccap_core_identification.CoreId = cap_info.CoreId
                                if cap_info.HasField("CoreIpAddress"):
                                    info_update.agent_info_update.ccap_core_identification.CoreIpAddress = \
                                        Convert.format_ip(coreIpAddress)
                                if cap_info.HasField("IsPrincipal"):
                                    info_update.agent_info_update.ccap_core_identification.IsPrincipal = cap_info.IsPrincipal
                                if cap_info.HasField("CoreName"):
                                    info_update.agent_info_update.ccap_core_identification.CoreName = cap_info.CoreName
                                if cap_info.HasField("VendorId"):
                                    info_update.agent_info_update.ccap_core_identification.VendorId = cap_info.VendorId
                                if cap_info.HasField("CoreMode"):
                                    info_update.agent_info_update.ccap_core_identification.CoreMode = cap_info.CoreMode
                                if cap_info.HasField("CoreFunction"):
                                    info_update.agent_info_update.ccap_core_identification.CoreFunction = cap_info.CoreFunction
                                if cap_info.HasField(
                                        "InitialConfigurationComplete"):
                                    info_update.agent_info_update.ccap_core_identification.InitialConfigurationComplete = cap_info.InitialConfigurationComplete
                                    if cap_info.InitialConfigurationComplete:
                                        status_changed, specific = self.handle_init_conf_completed(
                                            args)
                                        core_msg_str = "GCP_CFG_CPL"
                                if cap_info.HasField("MoveToOperational"):
                                    if cap_info.MoveToOperational:
                                        self._send_event_notification(
                                            core_id, protoDef.
                                            msg_core_event_notification.OK,
                                            "Move to Operational",
                                            "OPERATIONAL")

                                if cap_info.HasField("ResourceSetIndex"):
                                    info_update.agent_info_update.ccap_core_identification.ResourceSetIndex = cap_info.ResourceSetIndex
                                if cap_info.HasField("Index"):
                                    info_update.agent_info_update.ccap_core_identification.Index = cap_info.Index

                                ccap_core_instance = self.ccap_cores[core_id]
                                transport = self.mgrs[
                                    ccap_core_instance["mgr"]]['transport']
                                transport.sock.send(
                                    info_update.SerializeToString(),
                                    flags=zmq.NOBLOCK)
                                self.logger.debug(
                                    "Send info to id %s, msg:%s" %
                                    (core_id, info_update))
                elif descr.name == 'RedundantCoreIpAddress':
                    for ha_info in value:
                        must_field = [
                            'ActiveCoreIpAddress', 'StandbyCoreIpAddress',
                            'Operation'
                        ]
                        ret_field = filter(
                            lambda field: ha_info.HasField(field), must_field)
                        if len(must_field) != len(ret_field):
                            self.logger.warn(
                                "Received RPD HA message {} without must fields"
                                .format(ha_info))
                            return
                        caps = {
                            "ActiveCoreIpAddress":
                            Convert.format_ip(ha_info.ActiveCoreIpAddress),
                            "StandbyCoreIpAddress":
                            Convert.format_ip(ha_info.StandbyCoreIpAddress),
                            'interface':
                            interface_local,
                            "operation":
                            ha_info.Operation
                        }
                        for idx in self.mgrs:
                            event_request_rsp = protoDef.msg_event_notification(
                            )
                            event_request_rsp.mgr_event.mgr_id = idx
                            event_request_rsp.mgr_event.event_id = self.id
                            event_request_rsp.mgr_event.data = json.dumps(
                                "Ha/" + json.dumps(caps))
                            self.mgrs[idx]['transport'].sock.send(
                                event_request_rsp.SerializeToString(),
                                flags=zmq.NOBLOCK)
                            self.logger.debug(
                                "Send event notification to id %s, msg:%s" %
                                (idx, event_request_rsp))
                        # update the capabilities
                        if ha_info.Operation == ManagerProcess.OPERATION_CHANGE:
                            for _, s in self.process.orchestrator.sessions_active.items(
                            ):
                                addr = s.get_descriptor().addr_remote
                                if addr == caps["ActiveCoreIpAddress"]:
                                    if (hasattr(s.ccap_identification,
                                                "is_active") and
                                            s.ccap_identification.is_active):
                                        active_session = s
                                        active_session.ccap_identification.is_active = False
                                        self.logger.info(
                                            "HA CHANGE: set session[%s] to standby"
                                            % caps["StandbyCoreIpAddress"])
                                elif addr == caps["StandbyCoreIpAddress"]:
                                    if (hasattr(s.ccap_identification,
                                                "is_active") and
                                            not s.ccap_identification.is_active
                                        ):
                                        standby_session = s
                                        standby_session.ccap_identification.is_active = True
                                        self.logger.info(
                                            "HA CHANGE: set session[%s] to active"
                                            % caps["StandbyCoreIpAddress"])

                        elif ha_info.Operation == ManagerProcess.OPERATION_ADD:
                            for _, s in self.process.orchestrator.sessions_active.items(
                            ):
                                addr = s.get_descriptor().addr_remote
                                if addr == caps["ActiveCoreIpAddress"]:
                                    if (hasattr(s.ccap_identification,
                                                "is_active") and
                                            not s.ccap_identification.is_active
                                        ):
                                        self.logger.warn(
                                            "HA ADD: session[%s] is not active now"
                                            % caps["ActiveCoreIpAddress"])
                                elif addr == caps["StandbyCoreIpAddress"]:
                                    if (hasattr(s.ccap_identification,
                                                "is_active") and
                                            s.ccap_identification.is_active):
                                        self.logger.warn(
                                            "HA ADD: session[%s] is not inactive now"
                                            % caps["StandbyCoreIpAddress"])

                elif descr.name == 'ConfiguredCoreTable':
                    for cfg_table in value:
                        must_field = ['ConfiguredCoreIp', 'Operation']
                        ret_field = filter(
                            lambda field: cfg_table.HasField(field),
                            must_field)
                        if len(must_field) != len(ret_field):
                            self.logger.warn(
                                "Received RPD ConfiguredCoreTable message {} without must fields"
                                .format(cfg_table))
                            return
                        caps = {
                            "ccap_core":
                            Convert.format_ip(cfg_table.ConfiguredCoreIp),
                            'interface':
                            interface_local,
                            "operation":
                            cfg_table.Operation
                        }
                        for idx in self.mgrs:
                            event_request_rsp = protoDef.msg_event_notification(
                            )
                            event_request_rsp.mgr_event.mgr_id = idx
                            event_request_rsp.mgr_event.event_id = self.id
                            event_request_rsp.mgr_event.data = json.dumps(
                                "config_table/" + json.dumps(caps))
                            self.mgrs[idx]['transport'].sock.send(
                                event_request_rsp.SerializeToString(),
                                flags=zmq.NOBLOCK)
                            self.logger.debug(
                                "Send event notification to id %s, msg:%s" %
                                (idx, event_request_rsp))
                elif descr.name == 'MultiCore':
                    self.logger.info(
                        "Received RPD MultiCore ConfiguredCoreTable message")
                    for configuredCoreTable in value.ConfiguredCoreTable:
                        caps = {
                            "ccap_core":
                            Convert.format_ip(
                                configuredCoreTable.ConfiguredCoreIp),
                            'interface':
                            interface_local,
                            "operation":
                            ManagerProcess.OPERATION_ADD
                        }
                        for idx in self.mgrs:
                            event_request_rsp = protoDef.msg_event_notification(
                            )
                            event_request_rsp.mgr_event.mgr_id = idx
                            event_request_rsp.mgr_event.event_id = self.id
                            event_request_rsp.mgr_event.data = json.dumps(
                                "config_table/" + json.dumps(caps))
                            self.mgrs[idx]['transport'].sock.send(
                                event_request_rsp.SerializeToString(),
                                flags=zmq.NOBLOCK)
                            self.logger.debug(
                                "Send event multi-core tlv notification to id %s, msg:%s"
                                % (idx, event_request_rsp))
                elif descr.name == "ActivePrincipalCore":
                    # record the request info
                    core_ip = ''
                    if None is not args:
                        descr = session.get_descriptor()
                        core_ip = Convert.format_ip(descr.addr_remote)
                        self.rcp_req_group[(seq.seq_number, core_ip)] = \
                            (seq, session, transaction_identifier, trans_id, time())
                    for idx in self.mgrs:
                        event_request_rsp = protoDef.msg_event_notification()
                        event_request_rsp.mgr_event.mgr_id = idx
                        event_request_rsp.mgr_event.event_id = self.id
                        event_request_rsp.mgr_event.data = json.dumps(
                            "get_active_principal/" + str(seq.seq_number) +
                            ',' + core_ip)
                        self.mgrs[idx]['transport'].sock.send(
                            event_request_rsp.SerializeToString(),
                            flags=zmq.NOBLOCK)
                        self.logger.debug(
                            "Send event notification to id %s, msg:%s" %
                            (idx, event_request_rsp))
                else:
                    self.logger.info("Recv {} message {}".format(
                        descr.name, value))
                    return

        elif rcp_msg.RcpMessageType == t_RcpMessage.RPD_CONFIGURATION_DONE:
            self.logger.debug("Got configuration done message...")
            # changed the status, then send to corresponding requester
            status_changed, specific = self.handle_init_conf_completed(args)
            core_msg_str = "GCP_CFG_CPL"

        elif rcp_msg.RcpMessageType == t_RcpMessage.CONNECT_CLOSE_NOTIFICATION:
            specific = self._get_specific_core_para(rcp_msg, args)
            core_msg_str = "TCP_FAIL"
            status_changed = True
            if specific:
                reconnect = False
                if specific in self.rcp:
                    if self.rcp[specific]['status'] != self.DOWN:
                        self.rcp[specific]['status'] = self.DOWN
                        # notify mgr about connection info, mgr will log this,
                        # and send notify message to CCAP Core finally.
                        # TODO remove these code later
                        for idx in self.mgrs:
                            event_request_rsp = protoDef.msg_event_notification(
                            )
                            event_request_rsp.mgr_event.mgr_id = idx
                            event_request_rsp.mgr_event.event_id = self.id
                            event_request_rsp.mgr_event.data = json.dumps(
                                "connect_closed/" + specific[0] + ";" +
                                specific[1] + ";" + str(reconnect))
                            self.mgrs[idx]['transport'].sock.send(
                                event_request_rsp.SerializeToString(),
                                flags=zmq.NOBLOCK)
                            self.logger.debug(
                                "Send event notification to id %s, msg:%s" %
                                (idx, event_request_rsp))
        elif rcp_msg.RcpMessageType == t_RcpMessage.SESSION_INITIATED:
            specific = self._get_specific_core_para(rcp_msg, args)
            core_msg_str = "TCP_OK"
            status_changed = True
        elif rcp_msg.RcpMessageType == t_RcpMessage.IRA_RECEIVED:
            specific = self._get_specific_core_para(rcp_msg, args)
            core_msg_str = "GCP_IRA"
            status_changed = True
        elif rcp_msg.RcpMessageType == t_RcpMessage.REX_RECEIVED:
            specific = self._get_specific_core_para(rcp_msg, args)
            core_msg_str = "GCP_CFG"
            status_changed = True
        else:
            self.logger.error(
                "Unexpected IPC message received from "
                "RCP: type: %s(%u)",
                rcp_msg.t_RcpMessageType.Name(rcp_msg.RcpMessageType),
                rcp_msg.RcpMessageType)
            return

        # send the status change to the requester
        if not status_changed:
            return

        popup_list = list()
        if None is not specific and specific in self.rcp:
            self.rcp[specific]['lastChangeTime'] = time()
            for id in self.rcp[specific]["requester"]:
                if id not in self.ccap_cores:
                    popup_list.append(id)
                    continue
                event_request_rsp = protoDef.msg_event_notification()
                event_request_rsp.core_event.id = id
                event_request_rsp.core_event.ccap_core_id = id
                event_request_rsp.core_event.status = protoDef.msg_core_event_notification.OK
                event_request_rsp.core_event.reason = "Status changed"
                event_request_rsp.core_event.event_id = self.id
                event_request_rsp.core_event.result = core_msg_str if core_msg_str else self.rcp[
                    specific]['status']
                ccap_core = self.ccap_cores[id]
                transport = self.mgrs[ccap_core["mgr"]]['transport']
                transport.sock.send(event_request_rsp.SerializeToString(),
                                    flags=zmq.NOBLOCK)
                self.logger.info("Send status change to id %s, msg:%s" %
                                 (id, event_request_rsp))
            for idx in popup_list:
                self.rcp[specific]['requester'].remove(idx)
Example #4
0
    def handle_msg_eds_req(self, msg, slave, pkt):
        """Handles GCP EDS REQ message.

        :param msg: The GCP EDS REQ message
        :type msg: Message
        :param slave: The RCP slave sessions on which the message has been
         received.
        :type slave: RCPSlaveSession
        :param pkt: The RCP packet where the message was encapsulated
        :type pkt: RCPPacket
        :return:

        """
        seq_list = []
        for rcp_msg in msg.tlv_data.rcp_msgs:
            for seq in rcp_msg.sequences:

                # Handle special messages including redirect
                # and CCAP Capabilities and ssd
                if rcp_msg.rcp_message_id == rcp_tlv_def.RCP_MSG_TYPE_IRA:
                    if slave.is_ira_recv == False:
                        slave.is_ira_recv = True
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave,
                            seq,
                            pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val(),
                            msg_type=t_RcpMessage.IRA_RECEIVED)
                if rcp_msg.rcp_message_id == rcp_tlv_def.RCP_MSG_TYPE_REX:
                    if slave.is_rex_recv == False:
                        slave.is_rex_recv = True
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave,
                            seq,
                            pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val(),
                            msg_type=t_RcpMessage.REX_RECEIVED)
                if seq.parent_gpb.HasField("Ssd"):
                    self.logger.info("Handling Ssd received at %s" %
                                     slave.get_descriptor())
                    if (not slave.ccap_identification.is_active) \
                            or (not slave.ccap_identification.is_principal):
                        self.logger.debug(
                            "Ssd received from non active %d or non principal %d ",
                            slave.ccap_identification.is_active,
                            slave.ccap_identification.is_principal)
                        try:
                            resp = self.pkt_director.get_positive_rsp_packets(
                                slave, pkt)
                        except Exception as ex:
                            self.logger.warning(
                                "Got exception when constructing ssd rsp packet: %s",
                                str(ex))
                            raise
                        if 1 != len(resp):
                            raise RCPMSGHandlingError(
                                "Invalid packet response returned by director")
                        resp = resp[0]
                        try:
                            slave.io_ctx.add_tx_packet(resp)
                            self.logger.debug("send ssd response")
                        except GCPSessionFull:
                            self.logger.error(
                                "GCP session tx full, failed to send SSD response msg"
                            )
                            raise
                        continue

                if len(seq.parent_gpb.RpdRedirect) > 0:
                    self.logger.info("Handling redirect received at %s" %
                                     slave.get_descriptor())
                    # handle redirect and drop all next data
                    addr_list = []
                    for redir_item in seq.parent_gpb.RpdRedirect:
                        ip_addr = redir_item.RedirectIpAddress
                        addr_family = (socket.AF_INET if
                                       Convert.is_valid_ipv4_address(ip_addr)
                                       else socket.AF_INET6)
                        addr_list.append((ip_addr, addr_family))

                    # send redirect response
                    try:
                        resp = self.pkt_director.get_positive_rsp_packets(
                            slave, pkt)
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when constructing  redirect rsp packet: %s",
                            str(ex))
                        raise

                    if 1 != len(resp):
                        raise RCPMSGHandlingError(
                            "Invalid packet response returned by director")
                    resp = resp[0]

                    try:
                        slave.io_ctx.add_tx_packet(resp)
                    except GCPSessionFull:
                        self.logger.error(
                            "GCP session tx full, failed to send redirect response msg"
                        )
                        raise

                    slave.dispatcher.fd_modify(slave.get_socket_fd(),
                                               slave.dispatcher.MASK_WR_ERR)
                    self.logger.debug("Response to redirect added to TX queue")
                    try:
                        self.callback_set.redirect_received(slave, addr_list)
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling redirect msg: %s",
                            str(ex))
                        raise
                    continue

                if len(seq.parent_gpb.CcapCoreIdentification) > 0:
                    index = -1
                    identRecord = CcapCoreIdentification()
                    self.logger.info("Handling CcapCoreIdentification update")
                    ccap_caps = seq.parent_gpb.CcapCoreIdentification[0]
                    self.logger.debug("msg is: %s", ccap_caps)
                    op = seq.operation
                    if len(seq.parent_gpb.CcapCoreIdentification) > 1:
                        self.logger.warning(
                            "Only one instance of CCAP caps is expected, but received: %u",
                            len(seq.parent_gpb.CcapCoreIdentification))

                    core_ip = slave.get_descriptor().addr_remote

                    if ccap_caps.HasField("CoreIpAddress"):
                        core_ip = Convert.format_ip(ccap_caps.CoreIpAddress)
                        ip = Convert.format_ip(ccap_caps.CoreIpAddress)

                    if op == rcp_tlv_def.RCP_OPERATION_TYPE_WRITE:
                        if ccap_caps.HasField("Index"):
                            index = ccap_caps.Index
                            identRecord.index = index
                            identRecord.read()
                        else:
                            self.logger.warning(
                                "RCP write type %d should include index", op)
                            self.pkt_director.send_eds_response_directly(
                                slave, pkt.transaction_identifier,
                                msg.msg_fields.TransactionID.get_val(), seq,
                                False)
                            continue
                    elif op == rcp_tlv_def.RCP_OPERATION_TYPE_ALLOCATE_WRITE:
                        identRecord.allocateIndex(core_ip)
                        ccap_caps.Index = identRecord.index
                    elif op == rcp_tlv_def.RCP_OPERATION_TYPE_READ:
                        seq_list.append(seq)
                        continue

                    identRecord.core_ip_addr = core_ip
                    if ccap_caps.HasField("IsPrincipal"):
                        self.logger.info(
                            "Received NotifyRSP from CCAP core is_principal[%s]",
                            ccap_caps.IsPrincipal)
                        identRecord.is_principal = True if ccap_caps.IsPrincipal else False

                    self.logger.debug(
                        "CcapCoreIdentification operation=%d index=%d", op,
                        ccap_caps.Index)

                    if ccap_caps.HasField("CoreId"):
                        identRecord.core_id = ccap_caps.CoreId

                    if ccap_caps.HasField("CoreName"):
                        identRecord.core_name = ccap_caps.CoreName

                    if ccap_caps.HasField("VendorId"):
                        identRecord.vendor_id = ccap_caps.VendorId

                    if ccap_caps.HasField("CoreMode"):
                        identRecord.core_mode = ccap_caps.CoreMode
                        identRecord.is_active = ccap_caps.CoreMode is t_CcapCoreIdentification.COREMODEACTIVE

                    if ccap_caps.HasField("InitialConfigurationComplete"):
                        identRecord.initial_configuration_complete = ccap_caps.InitialConfigurationComplete

                    if ccap_caps.HasField("MoveToOperational"):
                        identRecord.move_to_operational = ccap_caps.MoveToOperational

                    if ccap_caps.HasField("CoreFunction"):
                        identRecord.core_function = ccap_caps.CoreFunction

                    if ccap_caps.HasField("ResourceSetIndex"):
                        identRecord.resource_set_index = ccap_caps.ResourceSetIndex

                    if op in [
                            rcp_tlv_def.RCP_OPERATION_TYPE_WRITE,
                            rcp_tlv_def.RCP_OPERATION_TYPE_ALLOCATE_WRITE
                    ]:
                        identRecord.write()
                        self.logger.debug(
                            "Core ident DB save index =%d core_ip_addr=%s op=%d",
                            identRecord.index, identRecord.core_ip_addr, op)
                    # Set the ccap core Identification into the slave session
                    slave.ccap_identification = identRecord

                    # call CCAP caps update callback
                    self.callback_set.ccap_identification_update(slave)

                    try:
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave, seq, pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val())
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling core identification msg: %s",
                            str(ex))
                        raise

                if seq.parent_gpb.HasField('RpdConfigurationDone'):
                    self.logger.info(
                        "Handling configuration done message to MGR")
                    self.pkt_director.send_eds_response_directly(
                        slave, pkt.transaction_identifier,
                        msg.msg_fields.TransactionID.get_val(), seq)
                    try:
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave,
                            seq,
                            pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val(),
                            msg_type=t_RcpMessage.RPD_CONFIGURATION_DONE)
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling cfg_done msg: %s",
                            str(ex))
                        raise

                    continue

                if seq.parent_gpb.HasField('RpdGlobal'):
                    if slave.ccap_identification.is_principal and slave.ccap_identification.is_active:
                        self.logger.info(
                            "Receive RpdGlobal message from active principal core via session %s",
                            slave.get_descriptor())
                    else:
                        self.logger.info(
                            "Receive RpdGlobal message from non active principal core via session %s",
                            slave.get_descriptor())
                        self.pkt_director.send_eds_response_directly(
                            slave, pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val(), seq)
                        continue

                if len(seq.parent_gpb.ConfiguredCoreTable) > 0:
                    self.logger.info(
                        "Handling configuration core table message to MGR")
                    self.pkt_director.send_eds_response_directly(
                        slave, pkt.transaction_identifier,
                        msg.msg_fields.TransactionID.get_val(), seq)
                    try:
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave, seq, pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val())
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling core_table msg: %s",
                            str(ex))
                        raise

                    continue

                if seq.parent_gpb.HasField('MultiCore'):
                    self.logger.info(
                        "Handling MultiCore configuration msg is_principal=%d is_active=%d",
                        slave.ccap_identification.is_principal,
                        slave.ccap_identification.is_active)
                    #                   ****** temporary 4 line hack follows to work around
                    #                   ****** the issue described in C3RPHY-122
                    resource_set_index = 0
                    for resource_set in seq.parent_gpb.MulitCore.ResourceSet:
                        resource_set.ResourceSetIndex = resource_set_index
                        resource_set_index = resource_set_index + 1


#                   ****** end of 4 line hack to get around C3RPHY-122
                    self.pkt_director.send_eds_response_directly(
                        slave, pkt.transaction_identifier,
                        msg.msg_fields.TransactionID.get_val(), seq)
                    try:
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave, seq, pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val())
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling  MultiCore msg: %s",
                            str(ex))
                        raise

                    continue

                if seq.parent_gpb.HasField('ActivePrincipalCore'):
                    self.logger.info(
                        "Handling get active principal request on session %s",
                        slave.get_descriptor())
                    try:
                        self.callback_set.configuration_to_rcp_wrapper(
                            slave, seq, pkt.transaction_identifier,
                            msg.msg_fields.TransactionID.get_val())
                    except Exception as ex:
                        self.logger.warning(
                            "Got exception when handling Active Principal core msg: %s",
                            str(ex))
                        raise

                    continue
                seq_list.append(seq)

        if not seq_list:
            self.logger.info("EDS message without any RCPSequence received")
            return

        try:
            self.callback_set.configuration_operation(
                session=slave,
                rcp_sequence_list=seq_list,
                pkt_req=pkt,
                gcp_msg=msg)
        except Exception as ex:
            # TODO we need to handle failures with a granularity
            self.logger.error("Failed to process configuration: %s", ex)
            import traceback
            self.logger.error(traceback.format_stack())
            raise RCPMSGHandlingError()