def mle_1_parameter(theta, r, f, a, b, MAX_ITERATIONS=10000):
    '''One parameter (Rausch) logistic ICC model parameter estimation using MLE
       From "Item Response Theory: Parameter Estimation Techniques"
       Chapter 2.3, pages 40-45 (esp Eqn 2.20) and
       Appendix A, pages 289-297 (port of A.3 on p.294)'''
    theta, r, f = map(np.asanyarray, [theta, r, f])  # ensure these are arrays
    p = r / f

    for i in range(MAX_ITERATIONS):
        print "iteration", i
        P = np.squeeze(logistic(dev_ab(a, b, theta)))
        W = P * (1 - P)
        W[np.where(W < W_CUTOFF)] = np.nan  # Delete any dud values

        dLdb = np.nansum(r - f * P)
        d2Ldb2 = -np.nansum(f * W)

        print 'dLdb', dLdb, 'd2L / db2', d2Ldb2

        # Plot the Log Likelihood & 1st&2nd derivatives
        '''
        import plt
        bs = np.arange(-10, 10, 0.1)
        Ps = [ logistic(dev_ab(a, b, theta))
              for b in bs ]
        L  = [ log_likelihood(theta, r, f, P)
               for P in Ps ]
        dLdb = [ np.nansum(r - f * P)
                for P in Ps ]
        d2Ldb2 = [ np.nansum(f * P * (1 - P))
                  for P in Ps ]
        plt.ioff()
        plt.plot(bs, L)
        plt.plot(bs, dLdb)
        plt.plot(bs, d2Ldb2)
        plt.show()
        '''

        # Db = rhs * np.linalg.inv(mat) # rhs = np.nansum(f * W * V), mat = np.nansum(f * W) V = (p - P) / W
        Db = dLdb / d2Ldb2

        b += Db

        print 'b', b, 'Db', Db

        if abs(b) > MAX_ZETA:
            raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)

        if abs(Db) <= .05:
            break

    if i == MAX_ITERATIONS:
        print "REACHED MAXIMUM NUMBER OF ITERATIONS"

    P = logistic(dev_ab(a, b, theta))
    chi2 = chiSquared(f, p, P)
    return a, b, chi2
def mle_2_parameter(theta, r, f, zeta, lam, MAX_ITERATIONS=10000):
    '''Two parameter logistic ICC model parameter estimation using MLE
       From "Item Response Theory: Parameter Estimation Techniques"
       Chapter 2.3, pages 40-45 (esp Eqn 2.20) and
       Appendix A, pages 289-297 (port of A.3 on p.294)'''
    theta, r, f = map(np.asanyarray, [theta, r, f])  # ensure these are arrays
    p = r / f

    for i in range(MAX_ITERATIONS):
        print "iteration", i
        P = np.squeeze(logistic(dev_zeta_lam(zeta, lam, theta)))
        W = P * (1 - P)
        W[np.where(W < W_CUTOFF)] = np.nan  # Delete any dud values
        V = (p - P) / W
        fW = f * W
        fWV = fW * V
        fWV2 = fWV * V
        fWth = fW * theta
        fWth2 = fWth * theta
        fWthV = fWth * V

        if np.nansum(fW) <= 0:
            raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)

        mat = np.nansum([[fW, fWth], [fWth, fWth2]], axis=-1)
        rhs = np.nansum([[fWV], [fWthV]], axis=-1)

        mat_det = np.linalg.det(mat)

        if mat_det == 0:
            raise ConvergenceError("Non-invertible matrix encountered %i" % i)

        Dzeta, Dlam = np.dot(np.linalg.inv(mat), rhs).flatten()

        if mat_det <= DM_CUTOFF:
            break

        zeta += Dzeta
        lam += Dlam

        if abs(zeta) > MAX_ZETA or abs(lam) > MAX_LAM:
            raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)

        if abs(Dzeta) <= .05 and abs(Dlam) <= .05:
            break

    if i == MAX_ITERATIONS:
        print "REACHED MAXIMUM NUMBER OF ITERATIONS"

    #chi2 = np.nansum(fWV2) # sum of f[i] * W[i] * v[i]^2
    P = logistic(dev_zeta_lam(zeta, lam, theta))
    chi2 = chiSquared(f, p, P)
    return zeta, lam, chi2
Beispiel #3
0
 def param_eval(self, d, ss, hps):
     """
     At distance dist, evaluate the prob of connection
     """
     p = util.logistic(d, ss['mu'], ss['lambda'])
     p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
     return p
Beispiel #4
0
 def param_eval(self, d, ss, hps):
     """
     At distance dist, evaluate the prob of connection
     """
     rate = util.logistic(d, ss['mu'], hps['lambda'])
     rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
     return rate
 def bayesian_predict(self, x):
     pred_var = np.sum(x * np.dot(x, self.covar), axis=1)
     pred_mean = np.dot(x, self.weights_map)
     kappa = (1 + np.pi * pred_var / 8)**(
         -0.5)  # as defined in Bishop's book chapter 4
     bayes_predictions = logistic(pred_mean * kappa)
     return bayes_predictions
Beispiel #6
0
 def read_outputs(self):
     last = self.values.shape[1] - 1
     r = np.dot(self.values[:, last], self.hidden_output_axons)
     r = util.logisticize_array(r)
     for i in range(0, self.outputs):
         r[i] = util.logistic(r[i])
     return r
Beispiel #7
0
 def param_eval(self, d, ss, hps):
     """
     At distance dist, evaluate the prob of connection
     """
     rate = util.logistic(d, ss['mu'], hps['lambda'])
     rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
     return rate
Beispiel #8
0
 def param_eval(self, d, ss, hps):
     """
     At distance dist, evaluate the prob of connection
     """
     p = util.logistic(d, ss['mu'], ss['lambda'])
     p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
     return p
def logistic_ll_hessian(weights, x):
    """Note the Hessian does not depend on the true labels."""
    output_probs = logistic(np.dot(x, weights))
    hessian = np.zeros([x.shape[1]] * 2, dtype=np.float64)
    for i in range(x.shape[0]):
        hessian -= output_probs[i] * (1 - output_probs[i]) * np.outer(
            x[i, :], x[i, :])
    return hessian
Beispiel #10
0
 def run_hidden(self):
     for i in range(0, self.hidden_depth-1):
         layer = self.values[:, i]
         next_layer = self.values[:, i+1]
         new_values = np.dot(layer, self.hidden_axons[i, :, :])
         self.values[:, i + 1] = np.add(self.values[:, i+1], new_values)
         for a in range(0, self.hidden_width):
             self.values[a, i + 1] = util.logistic(self.values[a, i+1])
Beispiel #11
0
    def sample_data(self, ss, hps):
        """
        NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from 
        suffstats. 
        """
        d = np.random.exponential(hps['mu_hp'])
        p = util.logistic(d, ss['mu'], ss['lambda'])
        p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
        link = np.random.rand() < p

        x = np.zeros(1, dtype=self.data_dtype())
        x[0]['distance'] = d
        x[0]['link'] = link
        return x[0]
Beispiel #12
0
 def sample_data(self, ss, hps):
     """
     NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from 
     suffstats. 
     """
     d = np.random.exponential(hps['mu_hp'])
     p = util.logistic(d, ss['mu'], ss['lambda'])
     p = p * (hps['p_max'] - hps['p_min']) + hps['p_min']
     link = np.random.rand() < p
     
     x = np.zeros(1, dtype=self.data_dtype())
     x[0]['distance'] = d
     x[0]['link'] = link
     return x[0]
Beispiel #13
0
 def sample_data(self, ss, hps):
     """
     NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from 
     suffstats. 
     """
     d = np.random.exponential(hps['mu_hp'])
     rate = util.logistic(d, ss['mu'], hps['lambda'])
     
     rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
     count = np.random.poisson(rate) 
     
     x = np.zeros(1, dtype=self.data_dtype())
     x[0]['distance'] = d
     x[0]['link'] = count
     return x[0]
Beispiel #14
0
    def sample_data(self, ss, hps):
        """
        NOTE THIS ONLY SAMPLES FROM THE PARAM P and not from 
        suffstats. 
        """
        d = np.random.exponential(hps['mu_hp'])
        rate = util.logistic(d, ss['mu'], hps['lambda'])

        rate = rate * (ss['rate_scale'] - hps['rate_min']) + hps['rate_min']
        count = np.random.poisson(rate)

        x = np.zeros(1, dtype=self.data_dtype())
        x[0]['distance'] = d
        x[0]['link'] = count
        return x[0]
Beispiel #15
0
def fit_and_evaluate_gaussian(split,
                              cond=False,
                              use_image_space=False,
                              return_avg=True):
    """
    Fits a gaussian to the train data and evaluates it on the given split.
    :param split: the data split to evaluate on, must be 'trn', 'val', or 'tst'
    :param cond: boolean, whether to fit a gaussian per conditional
    :param use_image_space: bool, whether to report log probability in [0, 1] image space (only for cifar and mnist)
    :param return_avg: bool, whether to return average log prob with std error, or all log probs
    :return: average log probability & standard error, or all lop probs
    """

    assert is_data_loaded(), 'Dataset hasn\'t been loaded'

    # choose which data split to evaluate on
    data_split = getattr(data, split, None)
    if data_split is None:
        raise ValueError('Invalid data split')

    if cond:
        comps = []
        for i in range(data.n_labels):
            idx = data.trn.labels == i
            comp = pdfs.fit_gaussian(data.trn.x[idx])
            comps.append(comp)
        prior = np.ones(data.n_labels, dtype=float) / data.n_labels
        model = pdfs.MoG(prior, xs=comps)

    else:
        model = pdfs.fit_gaussian(data.trn.x)

    logprobs = model.eval(data_split.x)

    if use_image_space:
        assert data_name in ['mnist', 'cifar10']
        z = util.logistic(data_split.x)
        logprobs += data.n_dims * np.log(1 - 2 * data.alpha) - np.sum(
            np.log(z) + np.log(1 - z), axis=1)

    if return_avg:
        avg_logprob = logprobs.mean()
        std_err = logprobs.std() / np.sqrt(data_split.N)
        return avg_logprob, std_err

    else:
        return logprobs
Beispiel #16
0
    def JL(params):
        count[0] += 1

        # unpack the parameters
        a, b, c = unpack_abc(A, B, C, params)

        Pstar = logistic(dev_ab(a, b, theta))
        P = scale_guessing(Pstar, c)

        if c == 0 and num_params < 3:
            Pstar = None  # optimize :)

        JLL = J(theta, r, f, P, a, b, c, Pstar)

        if return_history:
            abc_hist.append((a, b, c))

        if not use_2nd:
            return JLL

        HLL = H(theta, r, f, P, a, b, c, Pstar)

        return JLL, HLL
Beispiel #17
0
    def JL(params):
        count[0] += 1

        # unpack the parameters
        zeta, lam, c = unpack_zlc(ZETA, LAM, C, params)

        Pstar = logistic(dev_zeta_lam(zeta, lam, theta))
        P = scale_guessing(Pstar, c)

        if c == 0 and num_params < 3:
            Pstar = None  # optimize :)

        JLL = J(theta, r, f, P, zeta, lam, c, Pstar)

        if return_history:
            zlc_hist.append((zeta, lam, c))

        if not use_2nd:
            return JLL

        HLL = H(theta, r, f, P, zeta, lam, c, Pstar)

        return JLL, HLL
Beispiel #18
0
 def derivative_L(abc):
     _a, _b, _c = unpack_abc(a, b, c, abc)  # unpack the parameters
     Pstar = logistic(dev_ab(_a, _b, theta))
     P = scale_guessing(Pstar, _c)
     return DL(theta, r, f, P, _a, _b, _c, Pstar)
Beispiel #19
0
def plot_t1t1_params(fig, conn_and_dist, assign_vect, ss, hps, MAX_DIST=10, 
                     model="LogisticDistance", MAX_CLASSES = 20):
    """
    In the same order that we would plot the latent matrix, plot
    the per-parameter properties

    hps are per-relation hps

    note, tragically, this wants the whole figure

    """

    from mpl_toolkits.axes_grid1 import Grid
    from matplotlib import pylab
    assign_vect = np.array(assign_vect)
    canon_assign_vect = util.canonicalize_assignment(assign_vect)
    # create the mapping between existing and new
    canon_to_old  = {}
    for i, v in enumerate(canon_assign_vect):
        canon_to_old[v]= assign_vect[i]

    CLASSES = np.sort(np.unique(canon_assign_vect)) 
    
    CLASSN = len(CLASSES)

    if CLASSN > MAX_CLASSES:
        print "WARNING, TOO MANY CLASSES" 
        CLASSN = MAX_CLASSES

    img_grid = Grid(fig, 111, # similar to subplot(111)
                    nrows_ncols = (CLASSN, CLASSN),
                    axes_pad = 0.1,
                    add_all=True, 
                    share_all=True, 
                    label_mode = 'L',
                     )

    if "istance" not in model:
        return 

    for c1i, c1_canon in enumerate(CLASSES[:MAX_CLASSES]):
        for c2i, c2_canon in enumerate(CLASSES[:MAX_CLASSES]):
            c1 = canon_to_old[c1_canon]
            c2 = canon_to_old[c2_canon]
            ax_pos = c1i * CLASSN + c2i
            ax = img_grid[ax_pos]

            nodes_1 = np.argwhere(assign_vect == c1).flatten()
            nodes_2 = np.argwhere(assign_vect == c2).flatten()
            conn_dist_hist = []
            noconn_dist_hist = []
            flatten_dist_val = []
            assert len(nodes_1) > 0
            assert len(nodes_2) > 0 
            for n1 in nodes_1:
                for n2 in nodes_2:
                    d = conn_and_dist[n1, n2]['distance']
                    if conn_and_dist[n1, n2]['link']:
                        conn_dist_hist.append(d)
                    else:
                        noconn_dist_hist.append(d)
                    flatten_dist_val.append((d, conn_and_dist[n1, n2]['link']))
            flatten_dist_val = np.array(flatten_dist_val)
            bins = np.linspace(0, MAX_DIST, 20)
            fine_bins = np.linspace(0, MAX_DIST, 100)
            
            if model == "LogisticDistance" or model == "LogisticDistanceFixedLambda":
                # compute prob as a function of distance for this class
                htrue, _ = np.histogram(conn_dist_hist, bins)

                hfalse, _ = np.histogram(noconn_dist_hist, bins)

                p = htrue.astype(float) / (hfalse + htrue)

                ax.plot(bins[:-1], p, c='b', linewidth=3)


            if model == "LogisticDistance":
                c = ss[(c1, c2)]
                print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(bins), c
                y = util.logistic(fine_bins, c['mu'], c['lambda']) 
                y = y * (hps['p_max'] - hps['p_min']) + hps['p_min']
                ax.plot(fine_bins, y, c='r', linewidth=2) 
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 0.6, r"lamb: %3.2f" % c['lambda'], fontsize=4)
                ax.axvline(c['mu'], c='k')

            elif model == "LogisticDistanceFixedLambda":

                c = ss[(c1, c2)]
                y = util.logistic(fine_bins, c['mu'], hps['lambda']) 
                y = y * (c['p_scale'] - hps['p_min']) + hps['p_min']
                ax.plot(fine_bins, y, c='r') 
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 0.6, r"lamb: %3.2f" % hps['lambda'], fontsize=4)
                ax.axvline(c['mu'], c='k')

            elif model == "ExponentialDistancePoisson":
                if len(flatten_dist_val) > 0:
                    x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
                    y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
                    ax.scatter(flatten_dist_val[:, 0] + x_jitter, 
                               flatten_dist_val[:, 1] + y_jitter, 
                               edgecolor='none', 
                               s=2)
                c = ss[(c1, c2)]
                mu = c['mu']
                rate_scale = c['rate_scale']
                y = np.exp(-fine_bins/mu)
                y = y * rate_scale
                ax.plot(fine_bins, y, c='r') 
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 0.6, r"rate_scale: %3.2f" % c['rate_scale'], fontsize=4)
                ax.set_ylim(-1, 20.0)
                ax.axvline(c['mu'], c='k')

            elif model == "LogisticDistancePoisson":
                if len(flatten_dist_val) > 0:
                    x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
                    y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
                    ax.scatter(flatten_dist_val[:, 0] + x_jitter, 
                               flatten_dist_val[:, 1] + y_jitter, 
                               edgecolor='none', 
                               s=2)
                c = ss[(c1, c2)]
                y = util.logistic(fine_bins, c['mu'], hps['lambda']) 
                y = y * (c['rate_scale'] - hps['rate_min']) + hps['rate_min']
                ax.plot(fine_bins, y, c='r') 
                ax.text(0, 1, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 3, r"rate_scale: %3.2f" % c['rate_scale'], fontsize=4)
                ax.set_ylim(-1, 20.0)
                ax.axvline(c['mu'], c='k')

            elif model == "LinearDistance":
                print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(bins)
                c = ss[(c1, c2)]
                y = util.linear_dist(fine_bins, c['p'], c['mu']) 
                y += hps['p_min']
                ax.plot(fine_bins, y, c='r') 

            ax.set_xlim(0, MAX_DIST)
Beispiel #20
0
 def derivative_L(zlc):
     _zeta, _lam, _c = unpack_zlc(zeta, lam, c,
                                  zlc)  # unpack the parameters
     Pstar = logistic(dev_zeta_lam(_zeta, _lam, theta))
     P = scale_guessing(Pstar, _c)
     return DL(theta, r, f, P, _zeta, _lam, _c, Pstar)
Beispiel #21
0
def evaluate(model, split, n_samples=None):
    """
    Evaluate a trained model.
    :param model: the model to evaluate. Can be any made, maf, or real nvp
    :param split: string, the data split to evaluate on. Must be 'trn', 'val' or 'tst'
    :param n_samples: number of samples to generate from the model, or None for no samples
    """

    assert is_data_loaded(), 'Dataset hasn\'t been loaded'

    # choose which data split to evaluate on
    data_split = getattr(data, split, None)
    if data_split is None:
        raise ValueError('Invalid data split')

    if is_conditional(model):

        # calculate log probability
        logprobs = model.eval([data_split.y, data_split.x])
        print('logprob(x|y) = {0:.2f} +/- {1:.2f}').format(
            logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N))

        # classify test set
        logprobs = np.empty([data_split.N, data.n_labels])
        for i in xrange(data.n_labels):
            y = np.zeros([data_split.N, data.n_labels])
            y[:, i] = 1
            logprobs[:, i] = model.eval([y, data_split.x])
        predict_label = np.argmax(logprobs, axis=1)
        accuracy = (predict_label == data_split.labels).astype(float)
        logprobs = scipy.misc.logsumexp(logprobs, axis=1) - np.log(
            logprobs.shape[1])
        print('logprob(x) = {0:.2f} +/- {1:.2f}'.format(
            logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N)))
        print('classification accuracy = {0:.2%} +/- {1:.2%}'.format(
            accuracy.mean(), 2 * accuracy.std() / np.sqrt(data_split.N)))

        # generate data conditioned on label
        if n_samples is not None:
            for i in xrange(data.n_labels):

                # generate samples and sort according to log prob
                y = np.zeros(data.n_labels)
                y[i] = 1
                samples = model.gen(y, n_samples)
                lp_samples = model.eval([np.tile(y, [n_samples, 1]), samples])
                lp_samples = lp_samples[np.logical_not(np.isnan(lp_samples))]
                idx = np.argsort(lp_samples)
                samples = samples[idx][::-1]

                if data_name == 'mnist':
                    samples = (util.logistic(samples) -
                               data.alpha) / (1 - 2 * data.alpha)

                elif data_name == 'bsds300':
                    samples = np.hstack(
                        [samples, -np.sum(samples, axis=1)[:, np.newaxis]])

                elif data_name == 'cifar10':
                    samples = (util.logistic(samples) -
                               data.alpha) / (1 - 2 * data.alpha)
                    D = int(data.n_dims / 3)
                    r = samples[:, :D]
                    g = samples[:, D:2 * D]
                    b = samples[:, 2 * D:]
                    samples = np.stack([r, g, b], axis=2)

                else:
                    raise ValueError('non-image dataset')

                util.disp_imdata(samples, data.image_size, [5, 8])

    else:

        # calculate average log probability
        logprobs = model.eval(data_split.x)
        print('logprob(x) = {0:.2f} +/- {1:.2f}'.format(
            logprobs.mean(), 2 * logprobs.std() / np.sqrt(data_split.N)))

        # generate data
        if n_samples is not None:

            # generate samples and sort according to log prob
            samples = model.gen(n_samples)
            lp_samples = model.eval(samples)
            lp_samples = lp_samples[np.logical_not(np.isnan(lp_samples))]
            idx = np.argsort(lp_samples)
            samples = samples[idx][::-1]

            if data_name == 'mnist':
                samples = (util.logistic(samples) -
                           data.alpha) / (1 - 2 * data.alpha)

            elif data_name == 'bsds300':
                samples = np.hstack(
                    [samples, -np.sum(samples, axis=1)[:, np.newaxis]])

            elif data_name == 'cifar10':
                samples = (util.logistic(samples) -
                           data.alpha) / (1 - 2 * data.alpha)
                D = int(data.n_dims / 3)
                r = samples[:, :D]
                g = samples[:, D:2 * D]
                b = samples[:, 2 * D:]
                samples = np.stack([r, g, b], axis=2)

            else:
                raise ValueError('non-image dataset')

            util.disp_imdata(samples, data.image_size, [5, 8])

    plt.show()
Beispiel #22
0
def evaluate_logprob(model,
                     split,
                     use_image_space=False,
                     return_avg=True,
                     batch=2000):
    """
    Evaluate a trained model only in terms of log probability.
    :param model: the model to evaluate. Can be any made, maf, or real nvp
    :param split: string, the data split to evaluate on. Must be 'trn', 'val' or 'tst'
    :param use_image_space: bool, whether to report log probability in [0, 1] image space (only for cifar and mnist)
    :param return_avg: bool, whether to return average log prob with std error, or all log probs
    :param batch: batch size to use for computing log probability
    :return: average log probability & standard error, or all log probs
    """

    assert is_data_loaded(), 'Dataset hasn\'t been loaded'

    # choose which data split to evaluate on
    data_split = getattr(data, split, None)
    if data_split is None:
        raise ValueError('Invalid data split')

    if is_conditional(model):

        logprobs = np.empty([data_split.N, data.n_labels])

        for i in xrange(data.n_labels):

            # create labels
            y = np.zeros([data_split.N, data.n_labels])
            y[:, i] = 1

            # process data in batches to make sure they fit in memory
            r, l = 0, batch
            while r < data_split.N:
                logprobs[r:l, i] = model.eval([y[r:l], data_split.x[r:l]])
                l += batch
                r += batch

        logprobs = scipy.misc.logsumexp(logprobs, axis=1) - np.log(
            logprobs.shape[1])

    else:

        logprobs = np.empty(data_split.N)

        # process data in batches to make sure they fit in memory
        r, l = 0, batch
        while r < data_split.N:
            logprobs[r:l] = model.eval(data_split.x[r:l])
            l += batch
            r += batch

    if use_image_space:
        assert data_name in ['mnist', 'cifar10']
        z = util.logistic(data_split.x)
        logprobs += data.n_dims * np.log(1 - 2 * data.alpha) - np.sum(
            np.log(z) + np.log(1 - z), axis=1)

    if return_avg:
        avg_logprob = logprobs.mean()
        std_err = logprobs.std() / np.sqrt(data_split.N)
        return avg_logprob, std_err

    else:
        return logprobs
 def predict(self, x):
     return logistic(np.dot(x, self.weights))
Beispiel #24
0
def plot_t1t1_params(fig,
                     conn_and_dist,
                     assign_vect,
                     ss,
                     hps,
                     MAX_DIST=10,
                     model="LogisticDistance",
                     MAX_CLASSES=20):
    """
    In the same order that we would plot the latent matrix, plot
    the per-parameter properties

    hps are per-relation hps

    note, tragically, this wants the whole figure

    """

    from mpl_toolkits.axes_grid1 import Grid
    from matplotlib import pylab
    assign_vect = np.array(assign_vect)
    canon_assign_vect = util.canonicalize_assignment(assign_vect)
    # create the mapping between existing and new
    canon_to_old = {}
    for i, v in enumerate(canon_assign_vect):
        canon_to_old[v] = assign_vect[i]

    CLASSES = np.sort(np.unique(canon_assign_vect))

    CLASSN = len(CLASSES)

    if CLASSN > MAX_CLASSES:
        print "WARNING, TOO MANY CLASSES"
        CLASSN = MAX_CLASSES

    img_grid = Grid(
        fig,
        111,  # similar to subplot(111)
        nrows_ncols=(CLASSN, CLASSN),
        axes_pad=0.1,
        add_all=True,
        share_all=True,
        label_mode='L',
    )

    if "istance" not in model:
        return

    for c1i, c1_canon in enumerate(CLASSES[:MAX_CLASSES]):
        for c2i, c2_canon in enumerate(CLASSES[:MAX_CLASSES]):
            c1 = canon_to_old[c1_canon]
            c2 = canon_to_old[c2_canon]
            ax_pos = c1i * CLASSN + c2i
            ax = img_grid[ax_pos]

            nodes_1 = np.argwhere(assign_vect == c1).flatten()
            nodes_2 = np.argwhere(assign_vect == c2).flatten()
            conn_dist_hist = []
            noconn_dist_hist = []
            flatten_dist_val = []
            assert len(nodes_1) > 0
            assert len(nodes_2) > 0
            for n1 in nodes_1:
                for n2 in nodes_2:
                    d = conn_and_dist[n1, n2]['distance']
                    if conn_and_dist[n1, n2]['link']:
                        conn_dist_hist.append(d)
                    else:
                        noconn_dist_hist.append(d)
                    flatten_dist_val.append((d, conn_and_dist[n1, n2]['link']))
            flatten_dist_val = np.array(flatten_dist_val)
            bins = np.linspace(0, MAX_DIST, 20)
            fine_bins = np.linspace(0, MAX_DIST, 100)

            if model == "LogisticDistance" or model == "LogisticDistanceFixedLambda":
                # compute prob as a function of distance for this class
                htrue, _ = np.histogram(conn_dist_hist, bins)

                hfalse, _ = np.histogram(noconn_dist_hist, bins)

                p = htrue.astype(float) / (hfalse + htrue)

                ax.plot(bins[:-1], p, c='b', linewidth=3)

            if model == "LogisticDistance":
                c = ss[(c1, c2)]
                print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(
                    bins), c
                y = util.logistic(fine_bins, c['mu'], c['lambda'])
                y = y * (hps['p_max'] - hps['p_min']) + hps['p_min']
                ax.plot(fine_bins, y, c='r', linewidth=2)
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 0.6, r"lamb: %3.2f" % c['lambda'], fontsize=4)
                ax.axvline(c['mu'], c='k')

            elif model == "LogisticDistanceFixedLambda":

                c = ss[(c1, c2)]
                y = util.logistic(fine_bins, c['mu'], hps['lambda'])
                y = y * (c['p_scale'] - hps['p_min']) + hps['p_min']
                ax.plot(fine_bins, y, c='r')
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0, 0.6, r"lamb: %3.2f" % hps['lambda'], fontsize=4)
                ax.axvline(c['mu'], c='k')

            elif model == "ExponentialDistancePoisson":
                if len(flatten_dist_val) > 0:
                    x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
                    y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
                    ax.scatter(flatten_dist_val[:, 0] + x_jitter,
                               flatten_dist_val[:, 1] + y_jitter,
                               edgecolor='none',
                               s=2)
                c = ss[(c1, c2)]
                mu = c['mu']
                rate_scale = c['rate_scale']
                y = np.exp(-fine_bins / mu)
                y = y * rate_scale
                ax.plot(fine_bins, y, c='r')
                ax.text(0, 0.2, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0,
                        0.6,
                        r"rate_scale: %3.2f" % c['rate_scale'],
                        fontsize=4)
                ax.set_ylim(-1, 20.0)
                ax.axvline(c['mu'], c='k')

            elif model == "LogisticDistancePoisson":
                if len(flatten_dist_val) > 0:
                    x_jitter = np.random.normal(0, 0.01, len(flatten_dist_val))
                    y_jitter = np.random.normal(0, 0.05, len(flatten_dist_val))
                    ax.scatter(flatten_dist_val[:, 0] + x_jitter,
                               flatten_dist_val[:, 1] + y_jitter,
                               edgecolor='none',
                               s=2)
                c = ss[(c1, c2)]
                y = util.logistic(fine_bins, c['mu'], hps['lambda'])
                y = y * (c['rate_scale'] - hps['rate_min']) + hps['rate_min']
                ax.plot(fine_bins, y, c='r')
                ax.text(0, 1, r"mu: %3.2f" % c['mu'], fontsize=4)
                ax.text(0,
                        3,
                        r"rate_scale: %3.2f" % c['rate_scale'],
                        fontsize=4)
                ax.set_ylim(-1, 20.0)
                ax.axvline(c['mu'], c='k')

            elif model == "LinearDistance":
                print "MAX_DISTANCE=", MAX_DIST, np.max(fine_bins), np.max(
                    bins)
                c = ss[(c1, c2)]
                y = util.linear_dist(fine_bins, c['p'], c['mu'])
                y += hps['p_min']
                ax.plot(fine_bins, y, c='r')

            ax.set_xlim(0, MAX_DIST)
def mle_3_parameter(theta, r, f, a, b, c, MAX_ITERATIONS=10000):
    '''Three parameter logistic ICC model parameter estimation using MLE
       From "Item Response Theory: Parameter Estimation Techniques"
       Chapter 2.6, pages 48-57 (esp final eqn on p.55)

       For comparison,
       a = lambda
       -a * b = zeta

       (structure based on the 2PL function above)


       Proof that Qstar = 1 - Pstar:
       Q / Qstar = (1 - c) ()
       Q = (1 - c) * Qstar
       P = c + (1 - c) * Pstar
       Q = 1 - P = 1 - c - (1 - c) * Pstar
       So, 1 - c - (1 - c) * Pstar = (1 - c) * Qstar
       Qstar = ((1 - c) - (1 - c) * Pstar) / (1 - c) = 1 - Pstar'''
    theta, r, f = map(np.asanyarray, [theta, r, f])  # ensure these are arrays
    p = r / f

    for i in range(MAX_ITERATIONS):
        print "iteration", i
        aa = a * np.ones(f.shape)  # array version of a
        Pstar = logistic(dev_ab(a, b, theta))
        P = np.squeeze(scale_guessing(Pstar, c))
        pmP = p - P
        iPc = 1 / (P - c)
        Q = (1 - P)
        Qic = Q / (1 - c)
        rat = Pstar / P
        thmb = theta - b
        #W = Pstar * (1 - Pstar) / (P * (1 - P))
        #W[np.where(W<W_CUTOFF)] = np.nan # Delete any dud values
        #if np.any(np.nansum(f * W)<0):
        #    raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)

        # LL = np.sum(r * np.log(P)) + np.sum((f - r) * np.log(1 - P)) # Compute the log-likelihood
        L1, L2, L3 = np.array([thmb, -aa, iPc
                               ]) * f * pmP * rat  # This is right
        JLL = np.nansum([L1, L2, L3], axis=-1)
        rhs = np.nansum([[L1], [L2], [L3]], axis=-1)

        EL11, EL22, EL33 = np.array(
            [-P * Q * thmb**2 * rat, -a**2 * P * Q * rat, Qic * iPc]) * f * rat
        EL12, EL13, EL23 = np.array(
            [a * thmb * P * Q * rat, -thmb * Qic, a * Qic]) * f * rat

        # This was wrong, but somehow seemed to work just as well??
        #EL11, EL22, EL33 = np.array([-P * Q * thmb ** 2, -a**2 * P * Q, Qic * iPc]) * f * rat**2
        #EL12, EL13, EL23 = np.array([a * thmb * P * Q * rat, -thmb * Qic, a * Qic]) * f * rat

        mat = JJLL = np.nansum(
            [[EL11, EL12, EL13], [EL12, EL22, EL23], [EL13, EL23, EL33]],
            axis=-1)

        Da, Db, Dc = np.dot(np.linalg.inv(mat), rhs).flatten()

        if np.linalg.det(mat) <= DM_CUTOFF:
            break

        a += Da
        b += Db
        c += Dc

        if abs(a) > MAX_LAM or abs(b) > MAX_B or abs(c) > 1:
            raise ConvergenceError("OUT OF BOUNDS ERROR ITERATION %i" % i)

        if abs(Da) <= .05 and abs(Db) <= .05 and abs(Dc) <= .05:
            break

    if i == MAX_ITERATIONS:
        print "REACHED MAXIMUM NUMBER OF ITERATIONS"

    P = logistic3PL(a, b, c, theta)
    chi2 = chiSquared(f, p, P)
    return a, b, c, chi2
 def update_weights(self, x, y, lr):
     grad_log_lik = x.T.dot((y - logistic(x.dot(self.weights))))
     self.weights += lr * grad_log_lik
     return
def logistic_log_likelihood(weights, x, y):
    output_probs = logistic(np.dot(x, weights))
    return log_likelihood(y_true=y, y_pred=output_probs)
def logistic_ll_jacobian(weights, x, y):
    return x.T.dot((y - logistic(x.dot(weights))))
Beispiel #29
0
 def __predict_single_class(self, x, class_index):
     return logistic(self.__weight[class_index].dot(x) +
                     self.__bias[class_index])
Beispiel #30
0
 def __calc_gradient(self, theta, x, y):
     y_pred = logistic(x.dot(theta))
     _, data_size = x.shape
     return ((y_pred - y).dot(x)).transpose() / data_size
Beispiel #31
0
 def __calc_loss(self, theta, x, y):
     y_pred = logistic(x.dot(theta))
     return calc_loss(y_pred, y)
Beispiel #32
0
 def predict_prob(self, x):
     return logistic(self.weight.dot(x) + self.bias)
Beispiel #33
0
 def predict(self, x):
     return logistic(self.__weight.dot(x) + self.__bias)