Example #1
0
def trainLRbySGD(N_its,inst_generator, outfile, devkey, learning_rate=1e-4, regularizer=1e-2):
    weights = defaultdict(float)
    dv_acc = [None]*N_its
    tr_acc = [None]*N_its

    # this block is all to take care of regularization
    ratereg = learning_rate * regularizer
    def regularize(base_feats,t):
        for base_feat in base_feats:
            for label in ALL_LABELS:
                weights[(label,base_feat)] *= (1 - ratereg) ** (t-last_update[base_feat])
            last_update[base_feat] = t

    for it in xrange(N_its):
        tr_err = 0
        last_update = defaultdict(int) # reset, since we regularize at the end of every iteration
        for i,(inst,true_label) in enumerate(inst_generator):
            # apply "just-in-time" regularization to the weights for features in this instance
            regularize(inst,i)
            # compute likelihood gradient from this instance
            probs = computeLabelProbs(inst,weights,ALL_LABELS)
            if true_label != argmax(probs): tr_err += 1
            # your code for updating the weights goes here

        # regularize all features at the end of each iteration
        regularize([base_feature for label,base_feature in weights.keys()],i)
        
        dv_acc[it] = scorer.accuracy(evalClassifier(weights, outfile, devkey))
        tr_acc[it] = 1. - tr_err/float(i)
        print it,'dev:',dv_acc[it],'train:',tr_acc[it]
    return weights,tr_acc,dv_acc
Example #2
0
def find_best_smoother(x_tr, y_tr, x_dv, y_dv, smoothers):
    '''
    find the smoothing value that gives the best accuracy on the dev data

    :param x_tr: training instances
    :param y_tr: training labels
    :param x_dv: dev instances
    :param y_dv: dev labels
    :param smoothers: list of smoothing values
    :returns: best smoothing value
    :rtype: float

    '''
    score = {}
    for smoother in smoothers:
        theta_nb = estimate_nb(x_tr, y_tr, smoother)
        y_hat = clf_base.predict_all(x_dv, theta_nb, set(y_tr))
        score[smoother] = (evaluation.acc(y_hat, y_dv))
    return clf_base.argmax(score), score
Example #3
0
def trainLRbyAdaGrad(N_its,inst_generator, outfile, devkey, learning_rate=1e-4, regularizer=1e-2):
    weights = defaultdict(float)
    dv_acc = [None]*N_its
    tr_acc = [None]*N_its

    running_value = defaultdict(float)

    num_inst = len(inst_generator)
    # this block is all to take care of regularization
    ratereg = learning_rate * regularizer
    def regularize(base_feats, t):
        for base_feat in base_feats:
            for label in ALL_LABELS:
                weights[(label, base_feat)] *= (1 - ratereg) ** (t-last_update[base_feat])
            last_update[base_feat] = t

    for it in xrange(N_its):
        tr_err = 0
        last_update = defaultdict(int) # reset, since we regularize at the end of every iteration
        for i, (inst, true_label) in enumerate(inst_generator):
            # apply "just-in-time" regularization to the weights for features in this instance
            regularize(inst, i)
            # compute likelihood gradient from this instance
            probs = computeLabelProbs(inst, weights, ALL_LABELS)

            label_pred = argmax(probs)
            if true_label != label_pred:tr_err += 1

            for word, value in inst.items():
                weights[(true_label, word)] += num_inst * learning_rate * value / running_value.get((true_label, word), 1)
                for label in ALL_LABELS:
                    weights[(label, word)] -= num_inst * probs[label] * learning_rate * value / running_value.get((label, word), 1)
                running_value[(true_label, word)] = value**2

        # regularize all features at the end of each iteration
        regularize([base_feature for label,base_feature in weights.keys()], i)

        dv_acc[it] = scorer.accuracy(evalClassifier(weights, outfile, devkey))
        tr_acc[it] = 1. - tr_err/float(i)
        print it,'dev:',dv_acc[it],'train:',tr_acc[it]
    return weights,tr_acc,dv_acc
def find_best_smoother(x_tr, y_tr, x_dv, y_dv, smoothers):
    '''
    find the smoothing value that gives the best accuracy on the dev data

    :param x_tr: training instances
    :param y_tr: training labels
    :param x_dv: dev instances
    :param y_dv: dev labels
    :param smoothers: list of smoothing values
    :returns: best smoothing value
    :rtype: float

    '''
    accuracy = {}
    genres = set(y_dv)
    for smoother in smoothers:
        accuracy[smoother] = evaluation.acc(
            clf_base.predict_all(x_dv, estimate_nb(x_tr, y_tr, smoother),
                                 genres), y_dv)

    best_smoother = clf_base.argmax(accuracy)
    return best_smoother, accuracy
Example #5
0
def find_best_smoother(x_tr,y_tr,x_dv,y_dv,smoothers):
    """find the smoothing value that gives the best accuracy on the dev data

    :param x_tr: training instances
    :param y_tr: training labels
    :param x_dv: dev instances
    :param y_dv: dev labels
    :param smoothers: list of smoothing values to try
    :returns: best smoothing value, scores of all smoothing values
    :rtype: float, dict

    """
    labels = set(y_tr)
    smoother_scores = {}
    for smoother in smoothers:
        nb = estimate_nb(x_dv, y_dv, smoother)
        predictions = clf_base.predict_all(x_tr, nb, list(labels))
        score = 0
        for prediction, target in izip(predictions, y_tr):
            if prediction == target:
                score+= 1
        smoother_scores[smoother] = score
    return clf_base.argmax(smoother_scores), smoother_scores