Beispiel #1
0
class KernelizediFDD(Representation):

    """
    Kernelized version of iFDD
    """
    features = []
    candidates = {}
    # contains a set for each feature indicating the ids of
    base_id_sets = set()
                       # 1-dim features it refines
    base_feature_ids = []
    max_relevance = 0.

    def __init__(self, domain, kernel, active_threshold, discover_threshold,
                 kernel_args=[], normalization=True, sparsify=True,
                 max_active_base_feat=2, max_base_feat_sim=0.7):
        super(KernelizediFDD, self).__init__(domain)
        self.kernel = kernel
        self.kernel_args = kernel_args
        self.active_threshold = active_threshold
        self.discover_threshold = discover_threshold
        self.normalization = normalization
        self.sparsify = sparsify
        self.sorted_ids = PriorityQueueWithNovelty()
        self.max_active_base_feat = max_active_base_feat
        self.max_base_feat_sim = max_base_feat_sim
        self.candidates = {}
        self.features = []
        self.base_features_ids = []
        self.max_relevance = 0.
    def show_features(self):
        l = self.sorted_ids.toList()[:]
        key = lambda x: (
            len(self.features[x].base_ids),
            tuple(self.features[x].dim),
            tuple(self.features[x].center[self.features[x].dim]))
        l.sort(key=key)
        for i in l:
            f = self.features[i]
            print "{:>5} {:>20}".format(i, f)

    def plot_1d_features(self, dimension_idx=None):
        """Creates a plot for each specified dimension of the state space and shows
        all 1-dimensional features in this dimension
        If no indices are passed, all dimensions are plotted

        dimension_idx: either a single dimension index (int) or a list of indices.
        """
        idx = dimension_idx
        if isinstance(idx, int):
            idx = [idx]
        elif idx is None:
            idx = self.domain.continuous_dims

        feat_list = range(self.features_num)
        key = lambda x: (
            len(self.features[x].base_ids),
            tuple(self.features[x].dim),
            tuple(self.features[x].center[self.features[x].dim]))
        feat_list.sort(key=key)
        last_i = -1
        for k in feat_list:
            if len(self.features[k].dim) > 1:
                break
            cur_i = self.features[k].dim[0]
            if cur_i != last_i:
                if last_i in idx:
                    plt.draw()
                if cur_i in idx:
                    xi = np.linspace(
                        self.domain.statespace_limits[
                            cur_i,
                            0],
                        self.domain.statespace_limits[
                            cur_i,
                            1],
                        200)
                    x = np.zeros((200, self.domain.statespace_limits.shape[0]))
                    x[:, cur_i] = xi
                    plt.figure("Feature Dimension {}".format(cur_i))
            if cur_i in idx:
                y = [self.features[k].output(xk) for xk in x]
                plt.plot(x, y, label="id {}".format(k))
            last_i = cur_i
        plt.draw()

    def plot_2d_features(self, d1=None, d2=None, n_lines=3):
        """
        plot contours of all 2-dimensional features covering
        dimension d1 and d2. For each feature, n_lines number of lines
        are shown.
        If no dimensions are specified, the first two continuous dimensions
        are shown.

        d1, d2: indices of dimensions to show
        n_lines: number of countour lines per feature (default: 3)
        """
        if d1 is None and d2 is None:
            # just take the first two dimensions
            idx = self.domain.continuous_dims[:2]
        else:
            idx = [d1, d2]
        idx.sort()

        feat_list = range(self.features_num)
        key = lambda x: (
            len(self.features[x].base_ids),
            tuple(self.features[x].dim),
            tuple(self.features[x].center[self.features[x].dim]))
        feat_list.sort(key=key)
        last_i = -1
        last_j = -1
        for k in feat_list:
            if len(self.features[k].dim) < 2:
                continue
            elif len(self.features[k].dim) > 2:
                break
            cur_i = self.features[k].dim[0]
            cur_j = self.features[k].dim[1]
            if cur_i != last_i or cur_j != last_j:
                if last_i in idx and last_j in idx:
                    plt.draw()
                if cur_i in idx and cur_j in idx:
                    xi = np.linspace(
                        self.domain.statespace_limits[
                            cur_i,
                            0],
                        self.domain.statespace_limits[
                            cur_i,
                            1],
                        100)
                    xj = np.linspace(
                        self.domain.statespace_limits[
                            cur_j,
                            0],
                        self.domain.statespace_limits[
                            cur_j,
                            1],
                        100)
                    X, Y = np.meshgrid(xi, xj)
                    plt.figure(
                        "Feature Dimensions {} and {}".format(cur_i, cur_j))
            if cur_i in idx and cur_j in idx:
                Z = np.zeros_like(X)
                for m in xrange(100):
                    for n in xrange(100):
                        x = np.zeros(self.domain.statespace_limits.shape[0])
                        x[cur_i] = X[m, n]
                        x[cur_j] = Y[m, n]
                        Z[m, n] = self.features[k].output(x)
                plt.contour(X, Y, Z, n_lines)
            last_i = cur_i
            last_j = cur_j
        plt.draw()

    def plot_2d_feature_centers(self, d1=None, d2=None):
        """
        plot the centers of all 2-dimensional features covering
        dimension d1 and d2.
        If no dimensions are specified, the first two continuous dimensions
        are shown.

        d1, d2: indices of dimensions to show
        """
        if d1 is None and d2 is None:
            # just take the first two dimensions
            idx = self.domain.continuous_dims[:2]
        else:
            idx = [d1, d2]
        idx.sort()

        feat_list = range(self.features_num)
        key = lambda x: (
            len(self.features[x].base_ids),
            tuple(self.features[x].dim),
            tuple(self.features[x].center[self.features[x].dim]))
        feat_list.sort(key=key)
        last_i = -1
        last_j = -1
        for k in feat_list:
            if len(self.features[k].dim) < 2:
                continue
            elif len(self.features[k].dim) > 2:
                break
            cur_i = self.features[k].dim[0]
            cur_j = self.features[k].dim[1]
            if cur_i != last_i or cur_j != last_j:
                if last_i in idx and last_j in idx:
                    plt.draw()
                if cur_i in idx and cur_j in idx:
                    plt.figure(
                        "Feature Dimensions {} and {}".format(cur_i, cur_j))
            if cur_i in idx and cur_j in idx:
                plt.plot(
                    [self.features[k].center[cur_i]],
                    [self.features[k].center[cur_j]],
                    "r",
                    marker="x")
            last_i = cur_i
            last_j = cur_j
        plt.draw()

    def phi_nonTerminal(self, s):
        out = np.zeros(self.features_num)
        if not self.sparsify:
            for i in xrange(self.features_num):
                out[i] = self.features[i].output(s)
        else:
            # get all base feature values and check if they are activated
            active_bases = set([])
            for i in self.sorted_ids.toList()[::-1]:
                if len(self.features[i].base_ids) > 1:
                    break
                if self.features[i].output(s) >= self.active_threshold:
                    active_bases.add(i)

            base_vals = {k: 1. for k in active_bases}
            # iterate over the remaining compound features
            for i in self.sorted_ids.toList():
                if active_bases.issuperset(self.features[i].base_ids):
                    if self.sparsify > 1:
                        out[i] = self.features[i].output(s)
                        if self.sparsify > 2 or out[i] >= self.active_threshold:
                            active_bases -= self.features[i].base_ids
                    else:
                        u = 0
                        for k in self.features[i].base_ids:
                            u = max(u, base_vals[k])
                        out[i] = self.features[i].output(s) * u

                        for k in self.features[i].base_ids:
                            base_vals[k] -= out[i]
                            if base_vals[k] < 0:
                                active_bases.remove(k)

        if self.normalization:
            summ = out.sum()
            if summ != 0:
                out /= out.sum()
        return out

    def phi_raw(self, s, terminal):
        assert(terminal is False)
        out = np.zeros(self.features_num)
        for i in xrange(self.features_num):
            out[i] = self.features[i].output(s)
        return out

    #@profile
    def post_discover(self, s, terminal, a, td_error, phi_s=None):
        if phi_s is None:
            phi_s = self.phi(s, terminal)
        phi_s_unnorm = self.phi_raw(s, terminal)
        discovered = 0
        Q = self.Qs(s, terminal, phi_s=phi_s).reshape(-1, 1)
        # indices of active features
        active_indices = list(
            np.where(phi_s_unnorm > self.active_threshold)[0])
        # "active indices", active_indices
        # gather all dimensions regarded by active features
        active_dimensions = np.zeros((len(s)), dtype="int")
        closest_neighbor = np.zeros((len(s)))
        for i in active_indices:
            for j in self.features[i].dim:
                active_dimensions[j] += 1
                closest_neighbor[j] = max(closest_neighbor[j], phi_s_unnorm[i])

        # add new base features for all dimension not regarded
        for j in xrange(len(s)):
            if active_dimensions[j] < self.max_active_base_feat and (closest_neighbor[j] < self.max_base_feat_sim or active_dimensions[j] < 1):
                active_indices.append(self.add_base_feature(s, j, Q=Q))
                discovered += 1

        # update relevance statistics of all feature candidates
        if discovered:
            phi_s = self.phi(s, terminal)
        la = len(active_indices)
        if la * (la - 1) < len(self.candidates):
            for ind, cand in self.candidates.items():
                g, h = ind
                rel = self.update_relevance_stat(
                    cand,
                    g,
                    h,
                    td_error,
                    s,
                    a,
                    phi_s)
                self.max_relevance = max(rel, self.max_relevance)
                # add if relevance is high enough
                if rel > self.discover_threshold:
                    self.add_refined_feature(g, h, Q=Q)
                    discovered += 1

        else:
            # the result of both branches can be very different as this one
            # updates only combinations which are considered active.
            for g, h in combinations(active_indices, 2):
                # note: g, h are ordered as active_indices are ordered
                cand = self.candidates.get((g, h))
                if cand is None:
                    continue
                rel = self.update_relevance_stat(
                    cand,
                    g,
                    h,
                    td_error,
                    s,
                    a,
                    phi_s)
                self.max_relevance = max(rel, self.max_relevance)
                # add if relevance is high enough
                if rel > self.discover_threshold:
                    self.add_refined_feature(g, h, Q=Q)
                    discovered += 1

        if discovered:
            self.max_relevance = 0.
        return discovered

    def update_relevance_stat(
            self, candidate, index1, index2, td_error, s, a, phi_s):
        """
        make sure that inputs are ordered, i.e.,index1 <= index2!
        returns the relevance of a potential feature combination
        """
        candidate.td_error_sum += phi_s[index1] * phi_s[index2] * td_error
        candidate.activation_count += phi_s[index1] ** 2 * phi_s[index2] ** 2
        if candidate.activation_count == 0.:
            return 0.
        rel = np.abs(candidate.td_error_sum) / \
            np.sqrt(candidate.activation_count)
        return rel

    def add_base_feature(self, center, dim, Q):
        """
        adds a new 1-dimensional feature and returns its index
        """
        new_f = KernelizedFeature(
            center=center, dim=[dim], kernel_args=self.kernel_args,
            kernel=self.kernel, index=self.features_num)
        self.features.append(new_f)

        self.base_id_sets.add(new_f.base_ids)
        self.sorted_ids.push(-1, self.features_num)
        self.logger.debug(
            "Added Feature {} {}".format(
                self.features_num,
                new_f))

        # add combinations with all existing features as candidates
        new_cand = {(f, self.features_num): Candidate(f, self.features_num)
                    for f in xrange(self.features_num) if dim not in self.features[f].dim}

        self.candidates.update(new_cand)
        for f, _ in new_cand.keys():
            self.base_id_sets.add(new_f.base_ids | self.features[f].base_ids)
        self.features_num += 1

        # add parameter dimension
        if self.normalization:
            self.weight_vec = addNewElementForAllActions(
                self.weight_vec,
                self.domain.actions_num,
                Q)
        else:
            self.weight_vec = addNewElementForAllActions(
                self.weight_vec,
                self.domain.actions_num)
        return self.features_num - 1

    def add_refined_feature(self, index1, index2, Q):
        """
        adds the combination of 2 existing features to the representation
        """
        f1 = self.features[index1]
        f2 = self.features[index2]
        new_center = np.zeros_like(f1.center)
        cnt = np.zeros_like(f1.center)
        cnt[f1.dim] += 1
        cnt[f2.dim] += 1
        cnt[cnt == 0] = 1.
        new_center[f1.dim] += f1.center[f1.dim]
        new_center[f2.dim] += f2.center[f2.dim]
        new_center /= cnt
        new_dim = list(frozenset(f1.dim) | frozenset(f2.dim))
        new_base_ids = f1.base_ids | f2.base_ids
        new_dim.sort()
        new_f = KernelizedFeature(center=new_center, dim=new_dim,
                                  kernel_args=self.kernel_args,
                                  kernel=self.kernel, index=self.features_num,
                                  base_ids=new_base_ids)
        self.features.append(new_f)
        # Priority is the negative number of base ids
        self.sorted_ids.push(-len(new_f.base_ids), self.features_num)
        #assert(len(self.sorted_ids.toList()) == self.features_num + 1)
        self.base_id_sets.add(new_f.base_ids)
        del self.candidates[(index1, index2)]

        # add new candidates
        new_cand = {(f, self.features_num): Candidate(f, self.features_num) for f in xrange(self.features_num)
                    if (self.features[f].base_ids | new_base_ids) not in self.base_id_sets
                    and len(frozenset(self.features[f].dim) & frozenset(new_dim)) == 0}
        for c, _ in new_cand.keys():
            self.base_id_sets.add(new_base_ids | self.features[c].base_ids)
        self.candidates.update(new_cand)
        self.logger.debug(
            "Added refined feature {} {}".format(
                self.features_num,
                new_f))
        self.logger.debug("{} candidates".format(len(self.candidates)))
        self.features_num += 1
        if self.normalization:
            self.weight_vec = addNewElementForAllActions(
                self.weight_vec,
                self.domain.actions_num,
                Q)
        else:
            self.weight_vec = addNewElementForAllActions(
                self.weight_vec,
                self.domain.actions_num)

        return self.features_num - 1
Beispiel #2
0
class iFDD(Representation):
    ''' The incremental Feature Dependency Discovery Representation based on
    [Geramifard et al. 2011 ICML paper]. This representation starts with a set of given
    binary features and adds new features as the conjunction of existing features. Given n features
    iFDD can expand the set of features up to 2^n-1 features (i.e. conjunction of each subset of n
    features can be considered as a new feature.
    '''

    # It is a good starting point to see how relevances grow if threshold is
    # set to infinity.
    PRINT_MAX_RELEVANCE = False
    discovery_threshold = None  # psi in the paper
    # boolean specifying the use of the trick mentioned in the paper so that
    # features are getting sparser with more feature discoveries (i.e. use
    # greedy algorithm for feature activation)
    sparsify = None
    # dictionary mapping initial feature sets to iFDD_feature
    iFDD_features = None
    # dictionary mapping initial feature sets to iFDD_potential
    iFDD_potentials = None
    # dictionary mapping each feature index (ID) to its feature object
    featureIndex2feature = None
    debug = 0  # Print more stuff
    # dictionary mapping  initial active feature set phi_0(s) to its
    # corresponding active features at phi(s). Based on Tuna's Trick to speed
    # up iFDD
    cache = None
    # this should only increase speed. If results are different something is
    # wrong
    useCache = 0
    # Number of features to be expanded in the batch setting
    maxBatchDiscovery = 0
    # Minimum value of feature relevance for the batch setting
    batchThreshold = 0
    # ICML 11 iFDD would add sum of abs(TD-errors) while the iFDD plus uses
    # the abs(sum(TD-Error))/sqrt(potential feature presence count)
    iFDDPlus = 0
    # This is a priority queue based on the size of the features (Largest ->
    # Smallest). For same size features, it is also sorted based on the newest
    # -> oldest. Each element is the pointer to feature object.
    sortediFDDFeatures = None
    # A Representation that provides the initial set of features for iFDD
    initial_representation = None
    # Helper parameter to get a sense of appropriate threshold on the
    # relevance for discovery
    maxRelevance = -np.inf
    # As Christoph mentioned adding new features may affect the phi for all
    # states. This idea was to make sure both conditions for generating active
    # features generate the same result.
    use_chirstoph_ordered_features = True

    def __init__(self,
                 domain,
                 discovery_threshold,
                 initial_representation,
                 sparsify=True,
                 discretization=20,
                 debug=0,
                 useCache=0,
                 maxBatchDiscovery=1,
                 batchThreshold=0,
                 iFDDPlus=1,
                 seed=1):
        self.iFDD_features = {}
        self.iFDD_potentials = {}
        self.featureIndex2feature = {}
        self.cache = {}
        self.discovery_threshold = discovery_threshold
        self.sparsify = sparsify
        self.setBinsPerDimension(domain, discretization)
        self.features_num = initial_representation.features_num
        self.debug = debug
        self.useCache = useCache
        self.maxBatchDiscovery = maxBatchDiscovery
        self.batchThreshold = batchThreshold
        self.sortediFDDFeatures = PriorityQueueWithNovelty()
        self.initial_representation = initial_representation
        self.iFDDPlus = iFDDPlus
        self.isDynamic = True
        self.addInitialFeatures()
        super(iFDD, self).__init__(domain, discretization, seed)

    def phi_nonTerminal(self, s):
        """ Based on Tuna's Master Thesis 2012 """
        F_s = np.zeros(self.features_num, 'bool')
        F_s_0 = self.initial_representation.phi_nonTerminal(s)
        activeIndices = np.where(F_s_0 != 0)[0]
        if self.useCache:
            finalActiveIndices = self.cache.get(frozenset(activeIndices))
            if finalActiveIndices is None:
                # run regular and update the cache
                finalActiveIndices = self.findFinalActiveFeatures(
                    activeIndices)
        else:
            finalActiveIndices = self.findFinalActiveFeatures(activeIndices)
        F_s[finalActiveIndices] = 1
        return F_s

    def findFinalActiveFeatures(self, intialActiveFeatures):
        """
        Given the active indices of phi_0(s) find the final active indices of phi(s) based on discovered features
        """
        finalActiveFeatures = []
        k = len(intialActiveFeatures)
        initialSet = set(intialActiveFeatures)

        if 2**k <= self.features_num:
            # k can be big which can cause this part to be very slow
            # if k is large then find active features by enumerating on the
            # discovered features.
            if self.use_chirstoph_ordered_features:
                for i in range(k, 0, -1):
                    if len(initialSet) == 0:
                        break
                    # generate list of all combinations with i elements
                    cand_i = [(c, self.iFDD_features[frozenset(c)].index)
                              for c in combinations(initialSet, i)
                              if frozenset(c) in self.iFDD_features]
                    # sort (recent features (big ids) first)
                    cand_i.sort(key=lambda x: x[1], reverse=True)
                    # idx = -1
                    for candidate, ind in cand_i:
                        # the next block is for testing only
                        # cur_idx = self.iFDD_features[frozenset(candidate)].index
                        # if idx > 0:
                        #    assert(idx > cur_idx)
                        # idx = cur_idx

                        if len(initialSet) == 0:
                            # No more initial features to be mapped to extended
                            # ones
                            break

                        # This was missing from ICML 2011 paper algorithm.
                        # Example: [0,1,20], [0,20] is discovered, but if [0]
                        # is checked before [1] it will be added even though it
                        # is already covered by [0,20]
                        if initialSet.issuperset(set(candidate)):
                            feature = self.iFDD_features.get(
                                frozenset(candidate))
                            if feature is not None:
                                finalActiveFeatures.append(feature.index)
                                if self.sparsify:
                                    # print "Sets:", initialSet, feature.f_set
                                    initialSet = initialSet - feature.f_set
                                    # print "Remaining Set:", initialSet
            else:
                for candidate in powerset(initialSet, ascending=0):
                    if len(initialSet) == 0:
                        # No more initial features to be mapped to extended
                        # ones
                        break

                    # This was missing from ICML 2011 paper algorithm. Example:
                    # [0,1,20], [0,20] is discovered, but if [0] is checked
                    # before [1] it will be added even though it is already
                    # covered by [0,20]
                    if initialSet.issuperset(set(candidate)):
                        feature = self.iFDD_features.get(frozenset(candidate))
                        if feature is not None:
                            finalActiveFeatures.append(feature.index)
                            if self.sparsify:
                                # print "Sets:", initialSet, feature.f_set
                                initialSet = initialSet - feature.f_set
                                # print "Remaining Set:", initialSet
        else:
            # print "********** Using Alternative: %d > %d" % (2**k, self.features_num)
            # Loop on all features sorted on their size and then novelty and
            # activate features
            for feature in self.sortediFDDFeatures.toList():
                if len(initialSet) == 0:
                    # No more initial features to be mapped to extended ones
                    break

                if initialSet.issuperset(set(feature.f_set)):
                    finalActiveFeatures.append(feature.index)
                    if self.sparsify:
                        # print "Sets:", initialSet, feature.f_set
                        initialSet = initialSet - feature.f_set
                        # print "Remaining Set:", initialSet

        if self.useCache:
            self.cache[frozenset(intialActiveFeatures)] = finalActiveFeatures
        return finalActiveFeatures

    def post_discover(self, s, terminal, a, td_error, phi_s):
        """
        returns the number of added features
        """
        # Indices of non-zero elements of vector phi_s
        activeFeatures = phi_s.nonzero()[0]
        discovered = 0
        for g_index, h_index in combinations(activeFeatures, 2):
            discovered += self.inspectPair(g_index, h_index, td_error)
        return discovered

    def inspectPair(self, g_index, h_index, td_error):
        # Inspect feature f = g union h where g_index and h_index are the indices of features g and h
        # If the relevance is > Threshold add it to the list of features
        # Returns True if a new feature is added
        g = self.featureIndex2feature[g_index].f_set
        h = self.featureIndex2feature[h_index].f_set
        f = g.union(h)
        feature = self.iFDD_features.get(f)
        if not self.iFDDPlus:
            td_error = abs(td_error)

        if feature is not None:
            # Already exists
            return False

        # Look it up in potentials
        potential = self.iFDD_potentials.get(f)
        if potential is None:
            # Generate a new potential and put it in the dictionary
            potential = iFDD_potential(f, g_index, h_index)
            self.iFDD_potentials[f] = potential

        potential.cumtderr += td_error
        potential.cumabstderr += abs(td_error)
        potential.count += 1
        # Check for discovery

        if self.random_state.rand() < self.iFDDPlus:
            relevance = abs(potential.cumtderr) / np.sqrt(potential.count)
        else:
            relevance = potential.cumabstderr

        if relevance >= self.discovery_threshold:
            self.maxRelevance = -np.inf
            self.addFeature(potential)
            return True
        else:
            self.updateMaxRelevance(relevance)
            return False

    def show(self):
        self.showFeatures()
        self.showPotentials()
        self.showCache()

    def updateWeight(self, p1_index, p2_index):
        # Add a new weight corresponding to the new added feature for all actions.
        # The new weight is set to zero if sparsify = False, and equal to the
        # sum of weights corresponding to the parents if sparsify = True
        a = self.domain.actions_num
        # Number of feature before adding the new one
        f = self.features_num - 1
        if self.sparsify:
            newElem = (self.weight_vec[p1_index::f] +
                       self.weight_vec[p2_index::f]).reshape((-1, 1))
        else:
            newElem = None
        self.weight_vec = addNewElementForAllActions(self.weight_vec, a,
                                                     newElem)
        # We dont want to reuse the hased phi because phi function is changed!
        self.hashed_s = None

    def addInitialFeatures(self):
        for i in xrange(self.initial_representation.features_num):
            feature = iFDD_feature(i)
            # shout(self,self.iFDD_features[frozenset([i])].index)
            self.iFDD_features[frozenset([i])] = feature
            self.featureIndex2feature[feature.index] = feature
            # priority is 1/number of initial features corresponding to the
            # feature
            priority = 1
            self.sortediFDDFeatures.push(priority, feature)

    def addFeature(self, potential):
        # Add it to the list of features
        # Features_num is always one more than the max index (0-based)
        potential.index = self.features_num
        self.features_num += 1
        feature = iFDD_feature(potential)
        self.iFDD_features[potential.f_set] = feature
        # Expand the size of the weight_vec
        self.updateWeight(feature.p1, feature.p2)
        # Update the index to feature dictionary
        self.featureIndex2feature[feature.index] = feature
        # print "IN IFDD, New Feature = %d => Total Features = %d" % (feature.index, self.features_num)
        # Update the sorted list of features
        # priority is 1/number of initial features corresponding to the feature
        priority = 1 / (len(potential.f_set) * 1.)
        self.sortediFDDFeatures.push(priority, feature)

        # If you use cache, you should invalidate entries that their initial
        # set contains the set corresponding to the new feature
        if self.useCache:
            for initialActiveFeatures in self.cache.keys():
                if initialActiveFeatures.issuperset(feature.f_set):
                    if self.sparsify:
                        self.cache.pop(initialActiveFeatures)
                    else:
                        # If sparsification is not used, simply add the new
                        # feature id to all cached values that have feature set
                        # which is a super set of the features corresponding to
                        # the new discovered feature
                        self.cache[initialActiveFeatures].append(feature.index)
        if self.debug:
            self.show()

    def batchDiscover(self, td_errors, phi, states):
        # Discovers features using iFDD in batch setting.
        # TD_Error: p-by-1 (How much error observed for each sample)
        # phi: n-by-p features corresponding to all samples (each column corresponds to one sample)
        # self.batchThreshold is the minimum relevance value for the feature to
        # be expanded
        SHOW_PLOT = 0  # Shows the histogram of relevances
        maxDiscovery = self.maxBatchDiscovery
        n = self.features_num  # number of features
        p = len(td_errors)  # Number of samples
        counts = np.zeros((n, n))
        relevances = np.zeros((n, n))
        for i in xrange(p):
            phiphiT = np.outer(phi[i, :], phi[i, :])
            if self.iFDDPlus:
                relevances += phiphiT * td_errors[i]
            else:
                relevances += phiphiT * abs(td_errors[i])
            counts += phiphiT
        # Remove Diagonal and upper part of the relevances as they are useless
        relevances = np.triu(relevances, 1)
        non_zero_index = np.nonzero(relevances)
        if self.iFDDPlus:
            # Calculate relevances based on theoretical results of ICML 2013
            # potential submission
            relevances[non_zero_index] = np.divide(
                np.abs(relevances[non_zero_index]),
                np.sqrt(counts[non_zero_index]))
        else:
            # Based on Geramifard11_ICML Paper
            relevances[non_zero_index] = relevances[non_zero_index]

        # Find indexes to non-zero excited pairs
        # F1 and F2 are the parents of the potentials
        (F1, F2) = relevances.nonzero()
        relevances = relevances[F1, F2]
        if len(relevances) == 0:
            # No feature to add
            self.logger.debug("iFDD Batch: Max Relevance = 0")
            return False

        if SHOW_PLOT:
            e_vec = relevances.flatten()
            e_vec = e_vec[e_vec != 0]
            e_vec = np.sort(e_vec)
            plt.ioff()
            plt.plot(e_vec, linewidth=3)
            plt.show()

        # Sort based on relevances
        # We want high to low hence the reverse: [::-1]
        sortedIndices = np.argsort(relevances)[::-1]
        max_relevance = relevances[sortedIndices[0]]
        # Add top <maxDiscovery> features
        self.logger.debug(
            "iFDD Batch: Max Relevance = {0:g}".format(max_relevance))
        added_feature = False
        new_features = 0
        for j in xrange(len(relevances)):
            if new_features >= maxDiscovery:
                break
            max_index = sortedIndices[j]
            f1 = F1[max_index]
            f2 = F2[max_index]
            relevance = relevances[max_index]
            if relevance > self.batchThreshold:
                # print "Inspecting",
                # f1,f2,'=>',self.getStrFeatureSet(f1),self.getStrFeatureSet(f2)
                if self.inspectPair(f1, f2, np.inf):
                    self.logger.debug(
                        'New Feature %d: %s, Relevance = %0.3f' %
                        (self.features_num - 1,
                         self.getStrFeatureSet(self.features_num - 1),
                         relevances[max_index]))
                    new_features += 1
                    added_feature = True
            else:
                # Because the list is sorted, there is no use to look at the
                # others
                break
        return (
            # A signal to see if the representation has been expanded or not
            added_feature)

    def showFeatures(self):
        print "Features:"
        print "-" * 30
        print " index\t| f_set\t| p1\t| p2\t | Weights (per action)"
        print "-" * 30
        for feature in reversed(self.sortediFDDFeatures.toList()):
            # for feature in self.iFDD_features.itervalues():
            # print " %d\t| %s\t| %s\t| %s\t| %s" %
            # (feature.index,str(list(feature.f_set)),feature.p1,feature.p2,str(self.weight_vec[feature.index::self.features_num]))
            print " %d\t| %s\t| %s\t| %s\t| Omitted" % (
                feature.index, self.getStrFeatureSet(
                    feature.index), feature.p1, feature.p2)

    def showPotentials(self):
        print "Potentials:"
        print "-" * 30
        print " index\t| f_set\t| relevance\t| count\t| p1\t| p2"
        print "-" * 30
        for _, potential in self.iFDD_potentials.iteritems():
            print " %d\t| %s\t| %0.2f\t| %d\t| %s\t| %s" % (
                potential.index, str(np.sort(list(
                    potential.f_set))), potential.relevance, potential.count,
                potential.p1, potential.p2)

    def showCache(self):
        if self.useCache:
            print "Cache:"
            if len(self.cache) == 0:
                print 'EMPTY!'
                return
            print "-" * 30
            print " initial\t| Final"
            print "-" * 30
            for initial, active in self.cache.iteritems():
                print " %s\t| %s" % (str(list(initial)), active)

    def updateMaxRelevance(self, newRelevance):
        # Update a global max relevance and outputs it if it is updated
        if self.maxRelevance < newRelevance:
            self.maxRelevance = newRelevance
            if self.PRINT_MAX_RELEVANCE:
                self.logger.debug(
                    "iFDD Batch: Max Relevance = {0:g}".format(newRelevance))

    def getFeature(self, f_id):
        # returns a feature given a feature id
        if f_id in self.featureIndex2feature.keys():
            return self.featureIndex2feature[f_id]
        else:
            print "F_id %d is not valid" % f_id
            return None

    def getStrFeatureSet(self, f_id):
        # returns a string that corresponds to the set of features specified by
        # the given feature_id
        if f_id in self.featureIndex2feature.keys():
            return str(sorted(list(self.featureIndex2feature[f_id].f_set)))
        else:
            print "F_id %d is not valid" % f_id
            return None

    def featureType(self):
        return bool

    def __deepcopy__(self, memo):
        ifdd = iFDD(self.domain, self.discovery_threshold,
                    self.initial_representation, self.sparsify,
                    self.discretization, self.debug, self.useCache,
                    self.maxBatchDiscovery, self.batchThreshold, self.iFDDPlus)
        for s, f in self.iFDD_features.items():
            new_f = deepcopy(f)
            new_s = deepcopy(s)
            ifdd.iFDD_features[new_s] = new_f
            ifdd.featureIndex2feature[new_f.index] = new_f
        for s, p in self.iFDD_potentials.items():
            new_s = deepcopy(s)
            new_p = deepcopy(p)
            ifdd.iFDD_potentials[new_s] = deepcopy(new_p)
        ifdd.cache = deepcopy(self.cache)
        ifdd.sortediFDDFeatures = deepcopy(self.sortediFDDFeatures)
        ifdd.features_num = self.features_num
        ifdd.weight_vec = deepcopy(self.weight_vec)
        return ifdd
Beispiel #3
0
class iFDD(Representation):
    # It is a good starting point to see how relevances grow if threshold is
    # set to infinity.
    PRINT_MAX_RELEVANCE = False
    discovery_threshold = None  # psi in the paper
    # boolean specifying the use of the trick mentioned in the paper so that
    # features are getting sparser with more feature discoveries (i.e. use
    # greedy algorithm for feature activation)
    sparsify = None
    # dictionary mapping initial feature sets to iFDD_feature
    iFDD_features = None
    # dictionary mapping initial feature sets to iFDD_potential
    iFDD_potentials = None
    # dictionary mapping each feature index (ID) to its feature object
    featureIndex2feature = None
    debug = 0     # Print more stuff
    # dictionary mapping  initial active feature set phi_0(s) to its
    # corresponding active features at phi(s). Based on Tuna's Trick to speed
    # up iFDD
    cache = None
    # this should only increase speed. If results are different something is
    # wrong
    useCache = 0
    # Number of features to be expanded in the batch setting
    maxBatchDiscovery = 0
    # Minimum value of feature relevance for the batch setting
    batchThreshold = 0
    # ICML 11 iFDD would add sum of abs(TD-errors) while the iFDD plus uses
    # the abs(sum(TD-Error))/sqrt(potential feature presence count)
    iFDDPlus = 0
    # This is a priority queue based on the size of the features (Largest ->
    # Smallest). For same size features, it is also sorted based on the newest
    # -> oldest. Each element is the pointer to feature object.
    sortediFDDFeatures = None
    # A Representation that provides the initial set of features for iFDD
    initial_representation = None
    # Helper parameter to get a sense of appropriate threshold on the
    # relevance for discovery
    maxRelevance = -np.inf
    # As Christoph mentioned adding new features may affect the phi for all
    # states. This idea was to make sure both conditions for generating active
    # features generate the same result.
    use_chirstoph_ordered_features = True

    def __init__(
            self, domain, discovery_threshold, initial_representation, sparsify=True,
            discretization=20, debug=0, useCache=0, maxBatchDiscovery=1, batchThreshold=0, iFDDPlus=1):
        self.iFDD_features = {}
        self.iFDD_potentials = {}
        self.featureIndex2feature = {}
        self.cache = {}
        self.discovery_threshold = discovery_threshold
        self.sparsify = sparsify
        self.setBinsPerDimension(domain, discretization)
        self.features_num = initial_representation.features_num
        self.debug = debug
        self.useCache = useCache
        self.maxBatchDiscovery = maxBatchDiscovery
        self.batchThreshold = batchThreshold
        self.sortediFDDFeatures = PriorityQueueWithNovelty()
        self.initial_representation = initial_representation
        self.iFDDPlus = iFDDPlus
        self.isDynamic = True
        self.addInitialFeatures()
        super(iFDD, self).__init__(domain, discretization)

    def phi_nonTerminal(self, s):
        """ Based on Tuna's Master Thesis 2012 """
        F_s = np.zeros(self.features_num, 'bool')
        F_s_0 = self.initial_representation.phi_nonTerminal(
            s)
        activeIndices = np.where(F_s_0 != 0)[0]
        if self.useCache:
            finalActiveIndices = self.cache.get(frozenset(activeIndices))
            if finalActiveIndices is None:
                # run regular and update the cache
                finalActiveIndices = self.findFinalActiveFeatures(
                    activeIndices)
        else:
            finalActiveIndices = self.findFinalActiveFeatures(
                activeIndices)
        F_s[finalActiveIndices] = 1
        return F_s

    def findFinalActiveFeatures(self, intialActiveFeatures):
        """
        Given the active indices of phi_0(s) find the final active indices of phi(s) based on discovered features
        """
        finalActiveFeatures = []
        k = len(intialActiveFeatures)
        initialSet = set(intialActiveFeatures)

        if 2 ** k <= self.features_num:
            # k can be big which can cause this part to be very slow
            # if k is large then find active features by enumerating on the
            # discovered features.
            if self.use_chirstoph_ordered_features:
                for i in range(k, 0, -1):
                    if len(initialSet) == 0:
                        break
                    # generate list of all combinations with i elements
                    cand_i = [(c, self.iFDD_features[frozenset(c)].index)
                              for c in combinations(initialSet, i)
                              if frozenset(c) in self.iFDD_features]
                    # sort (recent features (big ids) first)
                    cand_i.sort(key=lambda x: x[1], reverse=True)
                    #idx = -1
                    for candidate, ind in cand_i:
                        # the next block is for testing only
                        #cur_idx = self.iFDD_features[frozenset(candidate)].index
                        # if idx > 0:
                        #    assert(idx > cur_idx)
                        #idx = cur_idx

                        if len(initialSet) == 0:
                            # No more initial features to be mapped to extended
                            # ones
                            break

                        # This was missing from ICML 2011 paper algorithm.
                        # Example: [0,1,20], [0,20] is discovered, but if [0]
                        # is checked before [1] it will be added even though it
                        # is already covered by [0,20]
                        if initialSet.issuperset(set(candidate)):
                            feature = self.iFDD_features.get(
                                frozenset(candidate))
                            if feature is not None:
                                finalActiveFeatures.append(feature.index)
                                if self.sparsify:
                                    # print "Sets:", initialSet, feature.f_set
                                    initialSet = initialSet - feature.f_set
                                    # print "Remaining Set:", initialSet
            else:
                for candidate in powerset(initialSet, ascending=0):
                    if len(initialSet) == 0:
                        # No more initial features to be mapped to extended
                        # ones
                        break

                    # This was missing from ICML 2011 paper algorithm. Example:
                    # [0,1,20], [0,20] is discovered, but if [0] is checked
                    # before [1] it will be added even though it is already
                    # covered by [0,20]
                    if initialSet.issuperset(set(candidate)):
                        feature = self.iFDD_features.get(frozenset(candidate))
                        if feature is not None:
                            finalActiveFeatures.append(feature.index)
                            if self.sparsify:
                                # print "Sets:", initialSet, feature.f_set
                                initialSet = initialSet - feature.f_set
                                # print "Remaining Set:", initialSet
        else:
            # print "********** Using Alternative: %d > %d" % (2**k, self.features_num)
            # Loop on all features sorted on their size and then novelty and
            # activate features
            for feature in self.sortediFDDFeatures.toList():
                if len(initialSet) == 0:
                    # No more initial features to be mapped to extended ones
                    break

                if initialSet.issuperset(set(feature.f_set)):
                    finalActiveFeatures.append(feature.index)
                    if self.sparsify:
                        # print "Sets:", initialSet, feature.f_set
                        initialSet = initialSet - feature.f_set
                        # print "Remaining Set:", initialSet

        if self.useCache:
            self.cache[frozenset(intialActiveFeatures)] = finalActiveFeatures
        return finalActiveFeatures

    def post_discover(self, s, terminal, a, td_error, phi_s):
        """
        returns the number of added features
        """
        # Indices of non-zero elements of vector phi_s
        activeFeatures = phi_s.nonzero()[0]
        discovered = 0
        for g_index, h_index in combinations(activeFeatures, 2):
            discovered += self.inspectPair(g_index, h_index, td_error)
        return discovered

    def inspectPair(self, g_index, h_index, td_error):
        # Inspect feature f = g union h where g_index and h_index are the indices of features g and h
        # If the relevance is > Threshold add it to the list of features
        # Returns True if a new feature is added
        g = self.featureIndex2feature[g_index].f_set
        h = self.featureIndex2feature[h_index].f_set
        f = g.union(h)
        feature = self.iFDD_features.get(f)
        if not self.iFDDPlus:
            td_error = abs(td_error)

        if feature is not None:
            # Already exists
            return False

        # Look it up in potentials
        potential = self.iFDD_potentials.get(f)
        if potential is None:
            # Generate a new potential and put it in the dictionary
            potential = iFDD_potential(f, g_index, h_index)
            self.iFDD_potentials[f] = potential

        potential.cumtderr += td_error
        potential.cumabstderr += abs(td_error)
        potential.count += 1
        # Check for discovery

        if np.random.rand() < self.iFDDPlus:
            relevance = abs(potential.cumtderr) / np.sqrt(potential.count)
        else:
            relevance = potential.cumabstderr

        if relevance >= self.discovery_threshold:
            self.maxRelevance = -np.inf
            self.addFeature(potential)
            return True
        else:
            self.updateMaxRelevance(relevance)
            return False

    def show(self):
        self.showFeatures()
        self.showPotentials()
        self.showCache()

    def updateWeight(self, p1_index, p2_index):
        # Add a new weight corresponding to the new added feature for all actions.
        # The new weight is set to zero if sparsify = False, and equal to the
        # sum of weights corresponding to the parents if sparsify = True
        a = self.domain.actions_num
        # Number of feature before adding the new one
        f = self.features_num - 1
        if self.sparsify:
            newElem = (self.weight_vec[p1_index::f] +
                       self.weight_vec[p2_index::f]).reshape((-1, 1))
        else:
            newElem = None
        self.weight_vec = addNewElementForAllActions(self.weight_vec, a, newElem)
        # We dont want to reuse the hased phi because phi function is changed!
        self.hashed_s = None

    def addInitialFeatures(self):
        for i in xrange(self.initial_representation.features_num):
            feature = iFDD_feature(i)
            # shout(self,self.iFDD_features[frozenset([i])].index)
            self.iFDD_features[frozenset([i])] = feature
            self.featureIndex2feature[feature.index] = feature
            # priority is 1/number of initial features corresponding to the
            # feature
            priority = 1
            self.sortediFDDFeatures.push(priority, feature)

    def addFeature(self, potential):
        # Add it to the list of features
        # Features_num is always one more than the max index (0-based)
        potential.index = self.features_num
        self.features_num += 1
        feature = iFDD_feature(potential)
        self.iFDD_features[potential.f_set] = feature
        # Expand the size of the weight_vec
        self.updateWeight(feature.p1, feature.p2)
        # Update the index to feature dictionary
        self.featureIndex2feature[feature.index] = feature
        # print "IN IFDD, New Feature = %d => Total Features = %d" % (feature.index, self.features_num)
        # Update the sorted list of features
        # priority is 1/number of initial features corresponding to the feature
        priority = 1 / (len(potential.f_set) * 1.)
        self.sortediFDDFeatures.push(priority, feature)

        # If you use cache, you should invalidate entries that their initial
        # set contains the set corresponding to the new feature
        if self.useCache:
            for initialActiveFeatures in self.cache.keys():
                if initialActiveFeatures.issuperset(feature.f_set):
                    if self.sparsify:
                        self.cache.pop(initialActiveFeatures)
                    else:
                        # If sparsification is not used, simply add the new
                        # feature id to all cached values that have feature set
                        # which is a super set of the features corresponding to
                        # the new discovered feature
                        self.cache[initialActiveFeatures].append(feature.index)
        if self.debug:
            self.show()

    def batchDiscover(self, td_errors, phi, states):
        # Discovers features using iFDD in batch setting.
        # TD_Error: p-by-1 (How much error observed for each sample)
        # phi: n-by-p features corresponding to all samples (each column corresponds to one sample)
        # self.batchThreshold is the minimum relevance value for the feature to
        # be expanded
        SHOW_PLOT = 0  # Shows the histogram of relevances
        maxDiscovery = self.maxBatchDiscovery
        n = self.features_num  # number of features
        p = len(td_errors)  # Number of samples
        counts = np.zeros((n, n))
        relevances = np.zeros((n, n))
        for i in xrange(p):
            phiphiT = np.outer(phi[i, :], phi[i,:])
            if self.iFDDPlus:
                relevances += phiphiT * td_errors[i]
            else:
                relevances += phiphiT * abs(td_errors[i])
            counts += phiphiT
        # Remove Diagonal and upper part of the relevances as they are useless
        relevances = np.triu(relevances, 1)
        non_zero_index = np.nonzero(relevances)
        if self.iFDDPlus:
            # Calculate relevances based on theoretical results of ICML 2013
            # potential submission
            relevances[non_zero_index] = np.divide(
                np.abs(relevances[non_zero_index]),
                np.sqrt(counts[non_zero_index]))
        else:
            # Based on Geramifard11_ICML Paper
            relevances[non_zero_index] = relevances[non_zero_index]

        # Find indexes to non-zero excited pairs
        # F1 and F2 are the parents of the potentials
        (F1, F2) = relevances.nonzero()
        relevances = relevances[F1, F2]
        if len(relevances) == 0:
            # No feature to add
            self.logger.debug("iFDD Batch: Max Relevance = 0")
            return False

        if SHOW_PLOT:
            e_vec = relevances.flatten()
            e_vec = e_vec[e_vec != 0]
            e_vec = np.sort(e_vec)
            plt.ioff()
            plt.plot(e_vec, linewidth=3)
            plt.show()

        # Sort based on relevances
        # We want high to low hence the reverse: [::-1]
        sortedIndices = np.argsort(relevances)[::-1]
        max_relevance = relevances[sortedIndices[0]]
        # Add top <maxDiscovery> features
        self.logger.debug(
            "iFDD Batch: Max Relevance = {0:g}".format(max_relevance))
        added_feature = False
        new_features = 0
        for j in xrange(len(relevances)):
            if new_features >= maxDiscovery:
                break
            max_index = sortedIndices[j]
            f1 = F1[max_index]
            f2 = F2[max_index]
            relevance = relevances[max_index]
            if relevance > self.batchThreshold:
                # print "Inspecting",
                # f1,f2,'=>',self.getStrFeatureSet(f1),self.getStrFeatureSet(f2)
                if self.inspectPair(f1, f2, np.inf):
                    self.logger.debug(
                        'New Feature %d: %s, Relevance = %0.3f' %
                        (self.features_num - 1, self.getStrFeatureSet(self.features_num - 1), relevances[max_index]))
                    new_features += 1
                    added_feature = True
            else:
                # Because the list is sorted, there is no use to look at the
                # others
                break
        return (
            # A signal to see if the representation has been expanded or not
            added_feature
        )

    def showFeatures(self):
        print "Features:"
        print "-" * 30
        print " index\t| f_set\t| p1\t| p2\t | Weights (per action)"
        print "-" * 30
        for feature in reversed(self.sortediFDDFeatures.toList()):
        # for feature in self.iFDD_features.itervalues():
            # print " %d\t| %s\t| %s\t| %s\t| %s" %
            # (feature.index,str(list(feature.f_set)),feature.p1,feature.p2,str(self.weight_vec[feature.index::self.features_num]))
            print " %d\t| %s\t| %s\t| %s\t| Omitted" % (feature.index, self.getStrFeatureSet(feature.index), feature.p1, feature.p2)

    def showPotentials(self):
        print "Potentials:"
        print "-" * 30
        print " index\t| f_set\t| relevance\t| count\t| p1\t| p2"
        print "-" * 30
        for _, potential in self.iFDD_potentials.iteritems():
            print " %d\t| %s\t| %0.2f\t| %d\t| %s\t| %s" % (potential.index, str(np.sort(list(potential.f_set))), potential.relevance, potential.count, potential.p1, potential.p2)

    def showCache(self):
        if self.useCache:
            print "Cache:"
            if len(self.cache) == 0:
                print 'EMPTY!'
                return
            print "-" * 30
            print " initial\t| Final"
            print "-" * 30
            for initial, active in self.cache.iteritems():
                print " %s\t| %s" % (str(list(initial)), active)

    def updateMaxRelevance(self, newRelevance):
        # Update a global max relevance and outputs it if it is updated
        if self.maxRelevance < newRelevance:
            self.maxRelevance = newRelevance
            if self.PRINT_MAX_RELEVANCE:
                self.logger.debug(
                    "iFDD Batch: Max Relevance = {0:g}".format(newRelevance))

    def getFeature(self, f_id):
        # returns a feature given a feature id
        if f_id in self.featureIndex2feature.keys():
            return self.featureIndex2feature[f_id]
        else:
            print "F_id %d is not valid" % f_id
            return None

    def getStrFeatureSet(self, f_id):
        # returns a string that corresponds to the set of features specified by
        # the given feature_id
        if f_id in self.featureIndex2feature.keys():
            return str(sorted(list(self.featureIndex2feature[f_id].f_set)))
        else:
            print "F_id %d is not valid" % f_id
            return None

    def featureType(self):
        return bool

    def __deepcopy__(self, memo):
        ifdd = iFDD(
            self.domain,
            self.discovery_threshold,
            self.initial_representation,
            self.sparsify,
            self.discretization,
            self.debug,
            self.useCache,
            self.maxBatchDiscovery,
            self.batchThreshold,
            self.iFDDPlus)
        for s, f in self.iFDD_features.items():
            new_f = deepcopy(f)
            new_s = deepcopy(s)
            ifdd.iFDD_features[new_s] = new_f
            ifdd.featureIndex2feature[new_f.index] = new_f
        for s, p in self.iFDD_potentials.items():
            new_s = deepcopy(s)
            new_p = deepcopy(p)
            ifdd.iFDD_potentials[new_s] = deepcopy(new_p)
        ifdd.cache = deepcopy(self.cache)
        ifdd.sortediFDDFeatures = deepcopy(self.sortediFDDFeatures)
        ifdd.features_num = self.features_num
        ifdd.weight_vec = deepcopy(self.weight_vec)
        return ifdd