def trainLRbySGD(N_its,inst_generator, outfile, devkey, learning_rate=1e-4, regularizer=1e-2):
weights = defaultdict(float)
dv_acc = [None]*N_its
tr_acc = [None]*N_its
# this block is all to take care of regularization
ratereg = learning_rate * regularizer
def regularize(base_feats,t):
for base_feat in base_feats:
for label in ALL_LABELS:
weights[(label,base_feat)] *= (1 - ratereg) ** (t-last_update[base_feat])
last_update[base_feat] = t
for it in xrange(N_its):
tr_err = 0
last_update = defaultdict(int) # reset, since we regularize at the end of every iteration
for i,(inst,true_label) in enumerate(inst_generator):
# apply "just-in-time" regularization to the weights for features in this instance
regularize(inst,i)
# compute likelihood gradient from this instance
probs = computeLabelProbs(inst,weights,ALL_LABELS)
if true_label != argmax(probs): tr_err += 1
# your code for updating the weights goes here
# regularize all features at the end of each iteration
regularize([base_feature for label,base_feature in weights.keys()],i)
dv_acc[it] = scorer.accuracy(evalClassifier(weights, outfile, devkey))
tr_acc[it] = 1. - tr_err/float(i)
print it,'dev:',dv_acc[it],'train:',tr_acc[it]
return weights,tr_acc,dv_acc
def find_best_smoother(x_tr, y_tr, x_dv, y_dv, smoothers):
'''
find the smoothing value that gives the best accuracy on the dev data
:param x_tr: training instances
:param y_tr: training labels
:param x_dv: dev instances
:param y_dv: dev labels
:param smoothers: list of smoothing values
:returns: best smoothing value
:rtype: float
'''
score = {}
for smoother in smoothers:
theta_nb = estimate_nb(x_tr, y_tr, smoother)
y_hat = clf_base.predict_all(x_dv, theta_nb, set(y_tr))
score[smoother] = (evaluation.acc(y_hat, y_dv))
return clf_base.argmax(score), score
def trainLRbyAdaGrad(N_its,inst_generator, outfile, devkey, learning_rate=1e-4, regularizer=1e-2):
weights = defaultdict(float)
dv_acc = [None]*N_its
tr_acc = [None]*N_its
running_value = defaultdict(float)
num_inst = len(inst_generator)
# this block is all to take care of regularization
ratereg = learning_rate * regularizer
def regularize(base_feats, t):
for base_feat in base_feats:
for label in ALL_LABELS:
weights[(label, base_feat)] *= (1 - ratereg) ** (t-last_update[base_feat])
last_update[base_feat] = t
for it in xrange(N_its):
tr_err = 0
last_update = defaultdict(int) # reset, since we regularize at the end of every iteration
for i, (inst, true_label) in enumerate(inst_generator):
# apply "just-in-time" regularization to the weights for features in this instance
regularize(inst, i)
# compute likelihood gradient from this instance
probs = computeLabelProbs(inst, weights, ALL_LABELS)
label_pred = argmax(probs)
if true_label != label_pred:tr_err += 1
for word, value in inst.items():
weights[(true_label, word)] += num_inst * learning_rate * value / running_value.get((true_label, word), 1)
for label in ALL_LABELS:
weights[(label, word)] -= num_inst * probs[label] * learning_rate * value / running_value.get((label, word), 1)
running_value[(true_label, word)] = value**2
# regularize all features at the end of each iteration
regularize([base_feature for label,base_feature in weights.keys()], i)
dv_acc[it] = scorer.accuracy(evalClassifier(weights, outfile, devkey))
tr_acc[it] = 1. - tr_err/float(i)
print it,'dev:',dv_acc[it],'train:',tr_acc[it]
return weights,tr_acc,dv_acc
def find_best_smoother(x_tr, y_tr, x_dv, y_dv, smoothers):
'''
find the smoothing value that gives the best accuracy on the dev data
:param x_tr: training instances
:param y_tr: training labels
:param x_dv: dev instances
:param y_dv: dev labels
:param smoothers: list of smoothing values
:returns: best smoothing value
:rtype: float
'''
accuracy = {}
genres = set(y_dv)
for smoother in smoothers:
accuracy[smoother] = evaluation.acc(
clf_base.predict_all(x_dv, estimate_nb(x_tr, y_tr, smoother),
genres), y_dv)
best_smoother = clf_base.argmax(accuracy)
return best_smoother, accuracy
def find_best_smoother(x_tr,y_tr,x_dv,y_dv,smoothers):
"""find the smoothing value that gives the best accuracy on the dev data
:param x_tr: training instances
:param y_tr: training labels
:param x_dv: dev instances
:param y_dv: dev labels
:param smoothers: list of smoothing values to try
:returns: best smoothing value, scores of all smoothing values
:rtype: float, dict
"""
labels = set(y_tr)
smoother_scores = {}
for smoother in smoothers:
nb = estimate_nb(x_dv, y_dv, smoother)
predictions = clf_base.predict_all(x_tr, nb, list(labels))
score = 0
for prediction, target in izip(predictions, y_tr):
if prediction == target:
score+= 1
smoother_scores[smoother] = score
return clf_base.argmax(smoother_scores), smoother_scores
