Esempio n. 1
0
class QosManager(object):
    qos_mgr = None

    @staticmethod
    def get_qos_manager(datapath, loop, config):
        if QosManager.qos_mgr:
            LOG.debug("Got QosManager instance")
            return QosManager.qos_mgr
        QosManager.qos_mgr = QosManager(datapath, loop, config)
        QosManager.qos_mgr.setup()
        return QosManager.qos_mgr

    @staticmethod
    def get_impl(datapath, loop, config):
        try:
            impl_type = QosImplType(config["qos"]["impl"])
        except ValueError:
            LOG.error("%s is not a valid qos impl type", impl_type)
            raise

        if impl_type == QosImplType.OVS_METER:
            return MeterManager(datapath, loop, config)
        else:
            return TCManager(datapath, loop, config)

    @classmethod
    def debug(cls, _, __, ___):
        config = load_service_config('pipelined')
        qos_impl_type = QosImplType(config["qos"]["impl"])
        qos_store = QosStore(cls.__name__)
        for k, v in qos_store.items():
            _, imsi, ip_addr, rule_num, d = get_key(k)
            _, qid, ambr, leaf = get_data(v)
            print('imsi :', imsi)
            print('ip_addr :', ip_addr)
            print('rule_num :', rule_num)
            print('direction :', d)
            print('qos_handle:', qid)
            print('qos_handle_ambr:', ambr)
            print('qos_handle_ambr_leaf:', leaf)
            if qos_impl_type == QosImplType.OVS_METER:
                MeterManager.dump_meter_state(v)
            else:
                intf = 'nat_iface' if d == FlowMatch.UPLINK else 'enodeb_iface'
                print("Dev: ", config[intf])
                TrafficClass.dump_class_state(config[intf], qid)
                if leaf and leaf != qid:
                    print("Leaf:")
                    TrafficClass.dump_class_state(config[intf], leaf)
                if ambr:
                    print("AMBR (parent):")
                    TrafficClass.dump_class_state(config[intf], ambr)

    def _is_redis_available(self):
        try:
            self._redis_store.client.ping()
        except ConnectionError:
            return False
        return True

    def __init__(self, datapath, loop, config):
        self._qos_enabled = config["qos"]["enable"]
        if not self._qos_enabled:
            return
        self._apn_ambr_enabled = config["qos"].get("apn_ambr_enabled", True)
        LOG.info("QoS: apn_ambr_enabled: %s", self._apn_ambr_enabled)
        self._clean_restart = config["clean_restart"]
        self._subscriber_state = {}
        self._loop = loop
        self.impl = QosManager.get_impl(datapath, loop, config)
        self._redis_store = QosStore(self.__class__.__name__)
        self._initialized = False
        self._redis_conn_retry_secs = 1

    def setup(self):
        if not self._qos_enabled:
            return

        if self._is_redis_available():
            return self._setupInternal()
        else:
            LOG.info("failed to connect to redis..retrying in %d secs",
                     self._redis_conn_retry_secs)
            self._loop.call_later(self._redis_conn_retry_secs, self.setup)

    def _setupInternal(self):
        if self._initialized:
            return
        if self._clean_restart:
            LOG.info("Qos Setup: clean start")
            self.impl.destroy()
            self._redis_store.clear()
            self.impl.setup()
            self._initialized = True
        else:
            # read existing state from qos_impl
            LOG.info("Qos Setup: recovering existing state")
            self.impl.setup()

            cur_qos_state, apn_qid_list = self.impl.read_all_state()
            LOG.debug("Initial qos_state -> %s",
                      json.dumps(cur_qos_state, indent=1))
            LOG.debug("apn_qid_list -> %s", apn_qid_list)
            LOG.debug("Redis state: %s", self._redis_store)
            try:
                # populate state from db
                in_store_qid = set()
                in_store_ambr_qid = set()
                purge_store_set = set()
                for rule, sub_data in self._redis_store.items():
                    _, qid, ambr, leaf = get_data(sub_data)
                    if qid not in cur_qos_state:
                        LOG.warning("missing qid: %s in TC", qid)
                        purge_store_set.add(rule)
                        continue
                    if ambr and ambr != 0 and ambr not in cur_qos_state:
                        purge_store_set.add(rule)
                        LOG.warning("missing ambr class: %s of qid %d", ambr,
                                    qid)
                        continue
                    if leaf and leaf != 0 and leaf not in cur_qos_state:
                        purge_store_set.add(rule)
                        LOG.warning("missing leaf class: %s of qid %d", leaf,
                                    qid)
                        continue

                    if ambr:
                        qid_state = cur_qos_state[qid]
                        if qid_state['ambr_qid'] != ambr:
                            purge_store_set.add(rule)
                            LOG.warning(
                                "Inconsistent amber class: %s of qid %d",
                                qid_state['ambr_qid'], ambr)
                            continue

                    in_store_qid.add(qid)
                    if ambr:
                        in_store_qid.add(ambr)
                        in_store_ambr_qid.add(ambr)
                    in_store_qid.add(leaf)

                    _, imsi, ip_addr, rule_num, direction = get_key(rule)

                    subscriber = self.get_or_create_subscriber(imsi)
                    subscriber.update_rule(ip_addr, rule_num, direction, qid,
                                           ambr, leaf)
                    session = subscriber.get_or_create_session(ip_addr)
                    session.set_ambr(direction, ambr, leaf)

                # purge entries from qos_store
                for rule in purge_store_set:
                    LOG.debug("purging qos_store entry %s", rule)
                    del self._redis_store[rule]

                # purge unreferenced qos configs from system
                # Step 1. Delete child nodes
                lost_and_found_apn_list = set()
                for qos_handle in cur_qos_state:
                    if qos_handle not in in_store_qid:
                        if qos_handle in apn_qid_list:
                            lost_and_found_apn_list.add(qos_handle)
                        else:
                            LOG.debug("removing qos_handle %d", qos_handle)
                            self.impl.remove_qos(
                                qos_handle,
                                cur_qos_state[qos_handle]['direction'],
                                recovery_mode=True)

                if len(lost_and_found_apn_list) > 0:
                    # Step 2. delete qos ambr without any leaf nodes
                    for qos_handle in lost_and_found_apn_list:
                        if qos_handle not in in_store_ambr_qid:
                            LOG.debug("removing apn qos_handle %d", qos_handle)
                            self.impl.remove_qos(
                                qos_handle,
                                cur_qos_state[qos_handle]['direction'],
                                recovery_mode=True,
                                skip_filter=True)
                final_qos_state, _ = self.impl.read_all_state()
                LOG.info("final_qos_state -> %s",
                         json.dumps(final_qos_state, indent=1))
                LOG.info("final_redis state -> %s", self._redis_store)
            except Exception as e:  # pylint: disable=broad-except
                # in case of any exception start clean slate

                LOG.error("error %s. restarting clean %s", e,
                          traceback.format_exc())
                self._clean_restart = True

            self._initialized = True

    def get_or_create_subscriber(self, imsi):
        subscriber_state = self._subscriber_state.get(imsi)
        if not subscriber_state:
            subscriber_state = SubscriberState(imsi, self._redis_store)
            self._subscriber_state[imsi] = subscriber_state
        return subscriber_state

    def add_subscriber_qos(
        self,
        imsi: str,
        ip_addr: str,
        apn_ambr: int,
        rule_num: int,
        direction: FlowMatch.Direction,
        qos_info: QosInfo,
    ):
        if not self._qos_enabled or not self._initialized:
            LOG.debug("add_subscriber_qos: not enabled or initialized")
            return None, None

        LOG.debug(
            "adding qos for imsi %s rule_num %d direction %d apn_ambr %d, qos_info %s",
            imsi, rule_num, direction, apn_ambr, qos_info)

        imsi = normalize_imsi(imsi)

        # ip_addr identifies a specific subscriber session, each subscriber session
        # must be associated with a default bearer and can be associated with dedicated
        # bearers. APN AMBR specifies the aggregate max bit rate for a specific
        # subscriber across all the bearers. Queues for dedicated bearers will be
        # children of default bearer Queues. In case the dedicated bearers exceed the
        # rate, then they borrow from the default bearer queue
        subscriber_state = self.get_or_create_subscriber(imsi)

        qos_handle = subscriber_state.get_qos_handle(rule_num, direction)
        if qos_handle:
            LOG.debug("qos exists for imsi %s rule_num %d direction %d", imsi,
                      rule_num, direction)

            return self.impl.get_action_instruction(qos_handle)

        ambr_qos_handle_root = 0
        ambr_qos_handle_leaf = 0
        if self._apn_ambr_enabled and apn_ambr > 0:
            session = subscriber_state.get_or_create_session(ip_addr)
            ambr_qos_handle_root = session.get_ambr(direction)
            LOG.debug("existing root rec: ambr_qos_handle_root %d",
                      ambr_qos_handle_root)

            if not ambr_qos_handle_root:
                ambr_qos_handle_root = self.impl.add_qos(direction,
                                                         QosInfo(gbr=None,
                                                                 mbr=apn_ambr),
                                                         skip_filter=True)
                if not ambr_qos_handle_root:
                    LOG.error(
                        'Failed adding root ambr qos mbr %u direction %d',
                        apn_ambr, direction)
                    return None, None
                else:
                    LOG.debug(
                        'Added root ambr qos mbr %u direction %d qos_handle %d ',
                        apn_ambr, direction, ambr_qos_handle_root)

            ambr_qos_handle_leaf = session.get_ambr_leaf(direction)
            LOG.debug("existing leaf rec: ambr_qos_handle_leaf %d",
                      ambr_qos_handle_leaf)

            if not ambr_qos_handle_leaf:
                ambr_qos_handle_leaf = self.impl.add_qos(
                    direction,
                    QosInfo(gbr=None, mbr=apn_ambr),
                    parent=ambr_qos_handle_root)
                if ambr_qos_handle_leaf:
                    session.set_ambr(direction, ambr_qos_handle_root,
                                     ambr_qos_handle_leaf)
                    LOG.debug(
                        'Added ambr qos mbr %u direction %d qos_handle %d/%d ',
                        apn_ambr, direction, ambr_qos_handle_root,
                        ambr_qos_handle_leaf)
                else:
                    LOG.error(
                        'Failed adding leaf ambr qos mbr %u direction %d',
                        apn_ambr, direction)
                    self.impl.remove_qos(ambr_qos_handle_root,
                                         direction,
                                         skip_filter=True)
                    return None, None
            qos_handle = ambr_qos_handle_leaf

        if qos_info:
            qos_handle = self.impl.add_qos(direction,
                                           qos_info,
                                           parent=ambr_qos_handle_root)
            LOG.debug("Added ded brr handle: %d", qos_handle)
            if qos_handle:
                LOG.debug('Adding qos %s direction %d qos_handle %d ',
                          qos_info, direction, qos_handle)
            else:
                LOG.error('Failed adding qos %s direction %d', qos_info,
                          direction)
                return None, None

        if qos_handle:
            subscriber_state.update_rule(ip_addr, rule_num, direction,
                                         qos_handle, ambr_qos_handle_root,
                                         ambr_qos_handle_leaf)
            return self.impl.get_action_instruction(qos_handle)
        return None, None

    def remove_subscriber_qos(self, imsi: str = "", rule_num: int = -1):
        if not self._qos_enabled or not self._initialized:
            LOG.debug("remove_subscriber_qos: not enabled or initialized")
            return

        LOG.debug("removing Qos for imsi %s rule_num %d", imsi, rule_num)
        if not imsi:
            LOG.error('imsi %s invalid, failed removing', imsi)
            return

        imsi = normalize_imsi(imsi)
        subscriber_state = self._subscriber_state.get(imsi)
        if not subscriber_state:
            LOG.debug('imsi %s not found, nothing to remove ', imsi)
            return

        to_be_deleted_rules = set()
        if rule_num == -1:
            # deleting all rules for the subscriber
            rules = subscriber_state.get_all_rules()
            for (rule_num, rule) in rules.items():
                session_with_rule = subscriber_state.find_session_with_rule(
                    rule_num)
                for (d, qos_data) in rule:
                    _, qid, _, leaf = get_data(qos_data)
                    if session_with_rule.get_ambr(d) != qid:
                        self.impl.remove_qos(qid, d)
                to_be_deleted_rules.add(rule_num)
        else:
            rule = subscriber_state.find_rule(rule_num)
            if rule:
                session_with_rule = subscriber_state.find_session_with_rule(
                    rule_num)
                for (d, qos_data) in rule:
                    _, qid, _, leaf = get_data(qos_data)
                    if session_with_rule.get_ambr(d) != qid:
                        self.impl.remove_qos(qid, d)
                        if leaf and leaf != qid:
                            self.impl.remove_qos(leaf, d)

                LOG.debug("removing rule %s %s ", imsi, rule_num)
                to_be_deleted_rules.add(rule_num)
            else:
                LOG.debug("unable to find rule_num %d for imsi %s", rule_num,
                          imsi)

        for rule_num in to_be_deleted_rules:
            subscriber_state.remove_rule(rule_num)

        # purge sessions with no rules
        for session in subscriber_state.get_all_empty_sessions():
            for d in (FlowMatch.UPLINK, FlowMatch.DOWNLINK):
                ambr_qos_handle = session.get_ambr(d)
                if ambr_qos_handle:
                    LOG.debug("removing root ambr qos handle %d direction %d",
                              ambr_qos_handle, d)
                    self.impl.remove_qos(ambr_qos_handle, d, skip_filter=True)
            LOG.debug("purging session %s %s ", imsi, session.ip_addr)
            subscriber_state.remove_session(session.ip_addr)

        # purge subscriber state with no rules
        if subscriber_state.check_empty():
            LOG.debug(
                "purging subscriber state for %s, empty rules and sessions",
                imsi)
            del self._subscriber_state[imsi]
Esempio n. 2
0
class QosManager(object):
    @staticmethod
    def get_impl(datapath, loop, config):
        try:
            impl_type = QosImplType(config["qos"]["impl"])
        except ValueError:
            LOG.error("%s is not a valid qos impl type", impl_type)
            raise

        if impl_type == QosImplType.OVS_METER:
            return MeterManager(datapath, loop, config)
        else:
            return TCManager(datapath, loop, config)

    @classmethod
    def debug(cls, _, __, ___):
        config = load_service_config('pipelined')
        qos_impl_type = QosImplType(config["qos"]["impl"])
        qos_store = QosStore(cls.__name__)
        for k, v in qos_store.items():
            _, imsi, ip_addr, rule_num, d = get_key(k)
            print('imsi :', imsi)
            print('ip_addr :', ip_addr)
            print('rule_num :', rule_num)
            print('direction :', d)
            print('qos_handle:', v)
            if qos_impl_type == QosImplType.OVS_METER:
                MeterManager.dump_meter_state(v)
            else:
                intf = 'nat_iface' if d == FlowMatch.UPLINK else 'enodeb_iface'
                TrafficClass.dump_class_state(config[intf], v)

    def redisAvailable(self):
        try:
            self._qos_store.client.ping()
        except ConnectionError:
            return False
        return True

    def __init__(self, datapath, loop, config):
        self._qos_enabled = config["qos"]["enable"]
        if not self._qos_enabled:
            return

        self._clean_restart = config["clean_restart"]
        self._subscriber_state = {}
        self._loop = loop
        self.impl = QosManager.get_impl(datapath, loop, config)
        self._qos_store = QosStore(self.__class__.__name__)
        self._initialized = False
        self._redis_conn_retry_secs = 1

    def setup(self):
        if not self._qos_enabled:
            return

        if self.redisAvailable():
            return self._setupInternal()
        else:
            LOG.info("failed to connect to redis..retrying in %d secs",
                     self._redis_conn_retry_secs)
            self._loop.call_later(self._redis_conn_retry_secs, self.setup)

    def _setupInternal(self):
        if self._clean_restart:
            LOG.info("Qos Setup: clean start")
            self.impl.destroy()
            self._qos_store.clear()
            self.impl.setup()
            self._initialized = True
        else:
            # read existing state from qos_impl
            LOG.info("Qos Setup: recovering existing state")
            self.impl.setup()

            def callback(fut):
                LOG.debug("read_all_state complete => \n%s", fut.result())
                qos_state = fut.result()
                try:
                    # populate state from db
                    in_store_qid = set()
                    purge_store_set = set()
                    for k, v in self._qos_store.items():
                        if v not in qos_state:
                            purge_store_set.add(k)
                            continue
                        in_store_qid.add(v)
                        _, imsi, ip_addr, rule_num, d = get_key(k)
                        subscriber = self.get_or_create_subscriber(imsi)
                        subscriber.update_rule(ip_addr, rule_num, d, v)

                        qid_state = qos_state[v]
                        if qid_state['ambr_qid'] != 0:
                            session = subscriber.get_or_create_session(ip_addr)
                            ambr_qid = qid_state['ambr_qid']
                            leaf = 0
                            # its AMBR QoS handle if its config matches with parent
                            # pylint: disable=too-many-function-args
                            if self.impl.same_qos_config(d, ambr_qid, v):
                                leaf = v
                            session.set_ambr(d, qid_state['ambr_qid'], leaf)

                    # purge entries from qos_store
                    for k in purge_store_set:
                        LOG.debug("purging qos_store entry %s qos_handle", k)
                        del self._qos_store[k]

                    # purge unreferenced qos configs from system
                    for qos_handle in qos_state:
                        if qos_handle not in in_store_qid:
                            LOG.debug("removing qos_handle %d", qos_handle)
                            self.impl.remove_qos(qos_handle,
                                qos_state[qos_handle]['direction'], recovery_mode=True)

                    self._initialized = True
                    LOG.info("init complete with state recovered successfully")
                except Exception as e:  # pylint: disable=broad-except
                    # in case of any exception start clean slate
                    LOG.error("error %s. restarting clean", str(e))
                    self._clean_restart = True
                    self.setup()

            asyncio.ensure_future(self.impl.read_all_state(), loop=self._loop).add_done_callback(callback)

    def get_or_create_subscriber(self, imsi):
        subscriber_state = self._subscriber_state.get(imsi)
        if not subscriber_state:
            subscriber_state = SubscriberState(imsi, self._qos_store)
            self._subscriber_state[imsi] = subscriber_state
        return subscriber_state

    def add_subscriber_qos(
        self,
        imsi: str,
        ip_addr: str,
        apn_ambr : int,
        rule_num: int,
        direction: FlowMatch.Direction,
        qos_info: QosInfo,
    ):
        if not self._qos_enabled or not self._initialized:
            LOG.error("add_subscriber_qos: not enabled or initialized")
            return None, None

        LOG.debug("adding qos for imsi %s rule_num %d direction %d apn_ambr %d, qos_info %s",
                   imsi, rule_num, direction, apn_ambr, qos_info)

        imsi = normalize_imsi(imsi)

        # ip_addr identifies a specific subscriber session, each subscriber session
        # must be associated with a default bearer and can be associated with dedicated
        # bearers. APN AMBR specifies the aggregate max bit rate for a specific
        # subscriber across all the bearers. Queues for dedicated bearers will be
        # children of default bearer Queues. In case the dedicated bearers exceed the
        # rate, then they borrow from the default bearer queue
        subscriber_state = self.get_or_create_subscriber(imsi)

        qos_handle = subscriber_state.get_qos_handle(rule_num, direction)
        LOG.debug("existing rec: qos_handle %d", qos_handle)
        if qos_handle:
            LOG.debug("qos exists for imsi %s rule_num %d direction %d",
                      imsi, rule_num, direction)
            return self.impl.get_action_instruction(qos_handle)

        ambr_qos_handle_root = None
        if apn_ambr > 0:
            session = subscriber_state.get_or_create_session(ip_addr)
            ambr_qos_handle_root = session.get_ambr(direction)
            LOG.debug("existing root rec: ambr_qos_handle_root %d", ambr_qos_handle_root)

            if not ambr_qos_handle_root:
                ambr_qos_handle_root = self.impl.add_qos(direction, QosInfo(gbr=None, mbr=apn_ambr))
                if not ambr_qos_handle_root:
                    LOG.error('Failed adding root ambr qos mbr %u direction %d',
                              apn_ambr, direction)
                    return None, None
                else:
                    LOG.debug('Added root ambr qos mbr %u direction %d qos_handle %d ',
                              apn_ambr, direction, ambr_qos_handle_root)

            ambr_qos_handle_leaf = session.get_ambr_leaf(direction)
            LOG.debug("existing leaf rec: ambr_qos_handle_leaf %d", ambr_qos_handle_leaf)

            if not ambr_qos_handle_leaf:
                ambr_qos_handle_leaf = self.impl.add_qos(direction,
                                                         QosInfo(gbr=None, mbr=apn_ambr),
                                                         parent=ambr_qos_handle_root)
                if ambr_qos_handle_leaf:
                    session.set_ambr(direction, ambr_qos_handle_root, ambr_qos_handle_leaf)
                    LOG.debug('Added ambr qos mbr %u direction %d qos_handle %d/%d ',
                              apn_ambr, direction, ambr_qos_handle_root, ambr_qos_handle_leaf)
                else:
                    LOG.error('Failed adding leaf ambr qos mbr %u direction %d',
                              apn_ambr, direction)
                    self.impl.remove_qos(ambr_qos_handle_root, direction)
                    return None, None
            qos_handle = ambr_qos_handle_leaf

        if qos_info:
            qos_handle = self.impl.add_qos(direction, qos_info, parent=ambr_qos_handle_root)
            LOG.debug("Added ded brr handle: %d", qos_handle)
            if qos_handle:
                LOG.debug('Adding qos %s direction %d qos_handle %d ',
                          qos_info, direction, qos_handle)
            else:
                LOG.error('Failed adding qos %s direction %d', qos_info, direction)
                return None, None

        LOG.debug("qos_handle %d", qos_handle)
        subscriber_state.update_rule(ip_addr, rule_num, direction, qos_handle)
        return self.impl.get_action_instruction(qos_handle)

    def remove_subscriber_qos(self, imsi: str = "", rule_num: int = -1):
        if not self._qos_enabled or not self._initialized:
            LOG.error("remove_subscriber_qos: not enabled or initialized")
            return

        LOG.debug("removing Qos for imsi %s rule_num %d", imsi, rule_num)
        if not imsi:
            LOG.error('imsi %s invalid, failed removing', imsi)
            return

        imsi = normalize_imsi(imsi)
        subscriber_state = self._subscriber_state.get(imsi)
        if not subscriber_state:
            LOG.debug('imsi %s not found, nothing to remove ', imsi)
            return

        to_be_deleted_rules = []
        if rule_num == -1:
            # deleting all rules for the subscriber
            rules = subscriber_state.get_all_rules()
            for (rule_num, rule) in rules.items():
                LOG.debug("removing rule %s %s ", imsi, rule_num)
                session_with_rule = subscriber_state.find_session_with_rule(rule_num)
                for (d, qos_handle) in rule:
                    if session_with_rule.get_ambr(d) != qos_handle:
                        self.impl.remove_qos(qos_handle, d)
                to_be_deleted_rules.append(rule_num)
        else:
            rule = subscriber_state.find_rule(rule_num)
            if rule is None:
                LOG.error("unable to find rule_num %d for imsi %s", rule_num, imsi)
                return

            session_with_rule = subscriber_state.find_session_with_rule(rule_num)
            for (d, qos_handle) in rule:
                if session_with_rule.get_ambr(d) != qos_handle:
                    self.impl.remove_qos(qos_handle, d)
            LOG.debug("removing rule %s %s ", imsi, rule_num)
            to_be_deleted_rules.append(rule_num)

        for rule_num in to_be_deleted_rules:
            subscriber_state.remove_rule(rule_num)

        # purge sessions with no rules
        for session in subscriber_state.get_all_empty_sessions():
            for d in (FlowMatch.UPLINK, FlowMatch.DOWNLINK):
                ambr_qos_handle = session.get_ambr(d)
                if ambr_qos_handle:
                    LOG.debug("removing root ambr qos handle %d direction %d", ambr_qos_handle, d)
                    self.impl.remove_qos(ambr_qos_handle, d)
            LOG.debug("purging session %s %s ", imsi, session.ip_addr)
            subscriber_state.remove_session(session.ip_addr)

        # purge subscriber state with no rules
        if subscriber_state.check_empty():
            LOG.debug("purging subscriber state for %s, empty rules and sessions", imsi)
            del self._subscriber_state[imsi]
Esempio n. 3
0
class QosManager(object):
    @staticmethod
    def getqos_impl(datapath, loop, config):
        try:
            qos_impl_type = QosImplType(config["qos"]["impl"])
        except ValueError:
            LOG.error("%s is not a valid qos impl type", qos_impl_type)
            raise

        if qos_impl_type == QosImplType.OVS_METER:
            return MeterManager(datapath, loop, config)
        else:
            return TCManager(datapath, loop, config)

    @classmethod
    def debug(cls, _):
        config = load_service_config('pipelined')
        qos_impl_type = QosImplType(config["qos"]["impl"])
        qos_store = QosStore(cls.__name__)
        for k, v in qos_store.items():
            _, imsi, rule_num, d = get_key(k)
            print('imsi :', imsi)
            print('rule_num :', rule_num)
            print('direction :', d)
            print('qos_handle:', v)
            if qos_impl_type == QosImplType.OVS_METER:
                MeterManager.dump_meter_state(v)
            else:
                intf = 'nat_iface' if d == FlowMatch.UPLINK else 'enodeb_iface'
                TrafficClass.dump_class_state(config[intf], v)

    def redisAvailable(self):
        try:
            self._qos_store.client.ping()
        except ConnectionError:
            return False
        return True

    def __init__(self, datapath, loop, config):
        # pylint: disable=unnecessary-lambda
        self._enable_qos = config["qos"]["enable"]
        if not self._enable_qos:
            return
        self.qos_impl = QosManager.getqos_impl(datapath, loop, config)
        self._loop = loop
        self._subscriber_map = defaultdict(lambda: defaultdict())
        self._clean_restart = config["clean_restart"]
        self._qos_store = QosStore(self.__class__.__name__)
        self._initialized = False
        self._redis_conn_retry_secs = 1

    def setup(self):
        if not self._enable_qos:
            return

        if self.redisAvailable():
            return self._setupInternal()
        else:
            LOG.info(
                "failed to connect to redis..retrying in %d secs",
                self._redis_conn_retry_secs,
            )
            self._loop.call_later(self._redis_conn_retry_secs, self.setup)

    def _setupInternal(self):
        LOG.info("Qos Setup")
        if self._clean_restart:
            LOG.info("clean start, wiping out existing state")
            self.qos_impl.destroy()
            self._qos_store.clear()
            self.qos_impl.setup()
            self._initialized = True
        else:
            # read existing state from qos_impl
            LOG.info("recovering existing state")

            def callback(fut):
                LOG.debug("read_all_state complete => \n%s", fut.result())
                qos_state = fut.result()
                try:
                    # populate state from db
                    in_store_qid = set()
                    purge_store_set = set()
                    for k, v in self._qos_store.items():
                        if v not in qos_state:
                            purge_store_set.add(k)
                            continue
                        in_store_qid.add(v)
                        _, imsi, rule_num, d = get_key(k)
                        if rule_num not in self._subscriber_map[imsi]:
                            self._subscriber_map[imsi][rule_num] = []
                        self._subscriber_map[imsi][rule_num].append((v, d))

                    # purge entries from qos_store
                    for k in purge_store_set:
                        LOG.debug("purging qos_store entry %s qos_handle", k)
                        del self._qos_store[k]

                    # purge unreferenced qos configs from system
                    for qos_handle, d in qos_state.items():
                        if qos_handle not in in_store_qid:
                            LOG.debug("removing qos_handle %d", qos_handle)
                            self.qos_impl.remove_qos(qos_handle,
                                                     d,
                                                     recovery_mode=True)

                    self._initialized = True
                    LOG.info("init complete with state recovered successfully")
                except Exception as e:  # pylint: disable=broad-except
                    # in case of any exception start clean slate
                    LOG.error("error %s. restarting clean", str(e))
                    self._clean_restart = True
                    self.setup()

            asyncio.ensure_future(self.qos_impl.read_all_state(),
                                  loop=self._loop).add_done_callback(callback)

    def add_subscriber_qos(
        self,
        imsi: str,
        rule_num: int,
        direction: FlowMatch.Direction,
        qos_info: QosInfo,
    ):
        if not self._enable_qos or not self._initialized:
            LOG.error(
                "add_subscriber_qos failed imsi %s rule_num %d \
                      direction %d failed qos not enabled or uninitialized",
                imsi,
                rule_num,
                direction,
            )
            return (None, None)

        imsi = normalizeIMSI(imsi)
        LOG.debug("adding qos for imsi %s rule_num %d", imsi, rule_num)
        k = get_subscriber_key(imsi, rule_num, direction)
        qos_handle = self._qos_store.get(get_json(k))
        if qos_handle:
            LOG.debug("qos handle already exists for %s", k)
            return self.qos_impl.get_action_instruction(qos_handle)

        qos_handle = self.qos_impl.add_qos(direction, qos_info)
        if rule_num not in self._subscriber_map[imsi]:
            self._subscriber_map[imsi][rule_num] = []

        self._subscriber_map[imsi][rule_num].append((qos_handle, direction))
        self._qos_store[get_json(k)] = qos_handle
        return self.qos_impl.get_action_instruction(qos_handle)

    def remove_subscriber_qos(self, imsi: str = "", rule_num: int = -1):
        if not self._enable_qos or not self._initialized:
            LOG.error(
                "remove_subscriber_qos failed imsi %s rule_num %d \
                      failed qos not enabled or uninitialized",
                imsi,
                rule_num,
            )
            return

        imsi = normalizeIMSI(imsi)
        LOG.debug("removing Qos for imsi %s rule_num %d", imsi, rule_num)
        if imsi:
            if imsi not in self._subscriber_map:
                LOG.debug("unable to find imsi %s", imsi)
                return

            if rule_num != -1:
                # delete queue associated with this rule
                if rule_num not in self._subscriber_map[imsi]:
                    LOG.error("unable to find rule_num %d for imsi %s",
                              rule_num, imsi)
                    return

                for (qos_handle,
                     direction) in self._subscriber_map[imsi][rule_num]:
                    self.qos_impl.remove_qos(qos_handle, direction)
                    del self._qos_store[get_json(
                        get_subscriber_key(imsi, rule_num, direction))]

                if len(self._subscriber_map[imsi]) == 1:
                    del self._subscriber_map[imsi]
                else:
                    del self._subscriber_map[imsi][rule_num]

            else:
                # delete all queues associated with this subscriber
                for rule_num, qd_list in self._subscriber_map[imsi].items():
                    for (qos_handle, direction) in qd_list:
                        self.qos_impl.remove_qos(qos_handle, direction)
                        del self._qos_store[get_json(
                            get_subscriber_key(imsi, rule_num, direction))]

                self._subscriber_map.pop(imsi)
        else:
            # delete Qos queues associated with all subscribers
            LOG.info("removing Qos for all subscribers")
            for imsi, rule_map in self._subscriber_map.items():
                for rule_num, qd_list in rule_map.items():
                    for (qos_handle, direction) in qd_list:
                        self.qos_impl.remove_qos(qos_handle, direction)
                        # delete from qos store
                        del self._qos_store[get_json(
                            get_subscriber_key(imsi, rule_num, direction))]
            self._subscriber_map.clear()