def run(self):
print(self.classifier_config)
logging.info('Training {0} with standard validation..'.format(
self.modelname))
regs = [10**t for t in range(-5, -1)] if self.usepytorch else \
[2**t for t in range(-2, 4, 1)]
if self.noreg:
regs = [1e-9 if self.usepytorch else 1e9]
scores = []
for reg in regs:
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=reg,
seed=self.seed,
cudaEfficient=self.cudaEfficient)
# TODO: Find a hack for reducing nb epoches in SNLI
clf.fit(self.X['train'],
self.y['train'],
validation_data=(self.X['valid'], self.y['valid']))
else:
clf = LogisticRegression(C=reg, random_state=self.seed)
clf.fit(self.X['train'], self.y['train'])
scores.append(
round(100 * clf.score(self.X['valid'], self.y['valid']), 2))
logging.info([('reg:' + str(regs[idx]), scores[idx])
for idx in range(len(scores))])
optreg = regs[np.argmax(scores)]
devaccuracy = np.max(scores)
logging.info('Validation : best param found is reg = {0} with score \
{1}'.format(optreg, devaccuracy))
clf = LogisticRegression(C=optreg, random_state=self.seed)
logging.info('Evaluating...')
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=optreg,
seed=self.seed,
cudaEfficient=self.cudaEfficient)
# TODO: Find a hack for reducing nb epoches in SNLI
clf.fit(self.X['train'],
self.y['train'],
validation_data=(self.X['valid'], self.y['valid']))
else:
clf = LogisticRegression(C=optreg, random_state=self.seed)
clf.fit(self.X['train'], self.y['train'])
yhat, yprobs = clf.predict(self.X['test'])
pred = []
probs = []
for i in yprobs:
probs.append(list(i))
print(probs)
for i in yhat:
pred.append(list(i))
print(pred)
testaccuracy = clf.score(self.X['test'], self.y['test'])
testaccuracy = round(100 * testaccuracy, 2)
return devaccuracy, testaccuracy, pred, probs
def run(self):
# cross-validation
logging.info('Training {0} with {1}-fold cross-validation'.format(
self.modelname, self.k))
regs = [10**t for t in range(-5, -1)] if self.usepytorch else \
[2**t for t in range(-1, 6, 1)]
skf = StratifiedKFold(n_splits=self.k,
shuffle=True,
random_state=self.seed)
scores = []
for reg in regs:
scanscores = []
for train_idx, test_idx in skf.split(self.train['X'],
self.train['y']):
# Split data
X_train, y_train = self.train['X'][train_idx], self.train['y'][
train_idx]
X_test, y_test = self.train['X'][test_idx], self.train['y'][
test_idx]
# Train classifier
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=reg,
seed=self.seed)
clf.fit(X_train, y_train, validation_data=(X_test, y_test))
else:
clf = LogisticRegression(C=reg, random_state=self.seed)
clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)
scanscores.append(score)
# Append mean score
scores.append(round(100 * np.mean(scanscores), 2))
# evaluation
logging.info([('reg:' + str(regs[idx]), scores[idx])
for idx in range(len(scores))])
optreg = regs[np.argmax(scores)]
devaccuracy = np.max(scores)
logging.info('Cross-validation : best param found is reg = {0} \
with score {1}'.format(optreg, devaccuracy))
logging.info('Evaluating...')
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=optreg,
seed=self.seed)
clf.fit(self.train['X'], self.train['y'], validation_split=0.05)
else:
clf = LogisticRegression(C=optreg, random_state=self.seed)
clf.fit(self.train['X'], self.train['y'])
yhat = clf.predict(self.test['X'])
testaccuracy = clf.score(self.test['X'], self.test['y'])
testaccuracy = round(100 * testaccuracy, 2)
return devaccuracy, testaccuracy, yhat
def run(self):
logging.info(
'Training {0} with (inner) {1}-fold cross-validation'.format(
self.modelname, self.k))
regs = [10**t for t in range(-5, -1)] if self.usepytorch else \
[2**t for t in range(-2, 4, 1)]
skf = StratifiedKFold(n_splits=self.k, shuffle=True, random_state=1111)
innerskf = StratifiedKFold(n_splits=self.k,
shuffle=True,
random_state=1111)
count = 0
for train_idx, test_idx in skf.split(self.X, self.y):
count += 1
X_train, X_test = self.X[train_idx], self.X[test_idx]
y_train, y_test = self.y[train_idx], self.y[test_idx]
scores = []
for reg in regs:
regscores = []
for inner_train_idx, inner_test_idx in innerskf.split(
X_train, y_train):
X_in_train, X_in_test = X_train[inner_train_idx], X_train[
inner_test_idx]
y_in_train, y_in_test = y_train[inner_train_idx], y_train[
inner_test_idx]
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=reg,
seed=self.seed)
clf.fit(X_in_train,
y_in_train,
validation_data=(X_in_test, y_in_test))
else:
clf = LogisticRegression(C=reg, random_state=self.seed)
clf.fit(X_in_train, y_in_train)
regscores.append(clf.score(X_in_test, y_in_test))
scores.append(round(100 * np.mean(regscores), 2))
optreg = regs[np.argmax(scores)]
logging.info('Best param found at split {0}: l2reg = {1} \
with score {2}'.format(count, optreg, np.max(scores)))
self.devresults.append(np.max(scores))
if self.usepytorch:
clf = MLP(self.classifier_config,
inputdim=self.featdim,
nclasses=self.nclasses,
l2reg=optreg,
seed=self.seed)
clf.fit(X_train, y_train, validation_split=0.05)
else:
clf = LogisticRegression(C=optreg, random_state=self.seed)
clf.fit(X_train, y_train)
yhat = clf.predict(X_test)
if (self.nclasses == 2):
self.f1results.append(round(100 * f1_score(yhat, y_test), 2))
self.testresults.append(round(100 * clf.score(X_test, y_test), 2))
devaccuracy = round(np.mean(self.devresults), 2)
testaccuracy = round(np.mean(self.testresults), 2)
if self.f1results:
testf1 = round(np.mean(self.f1results), 2)
else:
testf1 = None
return devaccuracy, testaccuracy, testf1