def evaluate(classifier, dataset, groundTruth, confusion=None, verbose=True):
"""Evaluate the classifier on the given dataset and returns the confusion matrix.
Uses only the points that are in the groundTruth parameter for the evaluation.
Parameters
----------
classifier : a function which given a point returns its class
dataset : the dataset from which to get the points
groundTruth : a map from the points to classify to their respective class
"""
progress = TextProgress(len(groundTruth))
done = 0
confusion = confusion or ConfusionMatrix()
for pointId, expected in groundTruth.items():
try:
found = classifier(dataset.point(pointId))
confusion.add(expected, found, pointId)
except Exception as e:
log.warning('Could not classify point "%s" because %s' %
(pointId, str(e)))
raise
done += 1
if verbose: progress.update(done)
return confusion
def evaluate(dataset, classifier, groundTruth):
'''evaluate the classifier on the given dataset, using only the points
that are in the groundTruth. It then returns the confusion matrix.
- dataset is the dataset from which to get the points
- classifier is a function which given a point returns its class
- groundTruth is a map from the points to classify to their respective class
'''
total = len(groundTruth)
done = 0
confusion = ConfusionMatrix()
for (pointId, expected) in groundTruth.items():
try:
found = classifier(dataset.point(pointId))
confusion.add(expected, found, pointId)
except Exception, e:
print 'WARNING: Could not classify point', pointId, 'because', str(
e)
done += 1
print '\r[%d%%]' % int(done * 100 / total),
def __init__(self, num_classes, normalized=False, ignore_index=None):
super(IoU, self).__init__()
self.conf_metric = ConfusionMatrix(num_classes, normalized)
if ignore_index is None:
self.ignore_index = None
elif isinstance(ignore_index, int):
self.ignore_index = (ignore_index, )
else:
try:
self.ignore_index = tuple(ignore_index)
except TypeError:
raise ValueError("'ignore_index' must be an int or iterable")
start = time.time()
y_test = []
y_pred = []
for j in range(len(data_test[i]["tweet"])):
prediction = nb.predict(data_test[i]["tweet"][j],
data_test[i]["target"][j])
y_test.append(data_test[i]["target"][j])
y_pred.append(prediction)
print("nb pred")
print(time.time() - start)
start = time.time()
cm = ConfusionMatrix()
accuracy, precision, recall, fmeasure = cm.score(y_test, y_pred)
print("Stopwords")
print(stopwords)
print("\nRemoved Stopwords")
print(removed_words)
print("\nAccuracy : {}".format(accuracy))
print("Precision : {}".format(precision))
print("Recall : {}".format(recall))
print("FMeasure : {}".format(fmeasure))
# df = pd.DataFrame({'X':x_array,'Y':y_array,'L':l_array,'K-Fold':kfold_per_combination,'Accuracy':list_acc,'Precision':list_prec,'Recall':list_recall,'F-Measure':list_fmeasure,'Fold Accuracy':fold_accuracy,'Fold Precision':fold_precision,'Fold Recall':fold_recall,'Fold F-Measure':fold_fmeasure})
# print(df)
# df.to_excel(r'cobabarunih.xlsx', index = False, header=True)
all_mon = train_out[3:]
grd_mon = train_out[:len(all_grads)]
upd_mon = train_out[len(all_grads):]
for pm, gm, um in zip(trainable_params, grd_mon, upd_mon):
if '.b' not in pm.name:
pad = (40-len(pm.name))*" "
print("%s \t %.5e \t %.5e \t %.5e" % (
pm.name + pad,
np.linalg.norm(gm),
np.linalg.norm(um),
np.linalg.norm(pm.get_value())
))
cost_train_lst += [cost_train]
conf_train = ConfusionMatrix(num_classes)
for i in range(x_train.shape[0] // 1000):
probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000])
preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1)
conf_train.batch_add(
y_train[i*1000:(i+1)*1000].flatten(),
preds_train_flat
)
if last_decay > args.decayinterval and epoch > args.nodecay:
last_decay = 0
old_lr = sh_lr.get_value(sh_lr)
new_lr = old_lr / args.decayfac
sh_lr.set_value(lasagne.utils.floatX(new_lr))
print("Decay lr from %f to %f" % (float(old_lr), float(new_lr)))
else:
targetspace[tv[0]], tv[1])
logger("Done training files.", monitor)
if outputmodel:
# output character patterns to be able to generate new tweetvectors for separate testing on trained data
stringspace.saveelementspace(charactervectorspacefilename)
# output model here with info about the category of each model item
with open(categorymodelfilename, "wb") as outfile:
pickle.dump(targetspace, outfile)
logger(
"Testing targetspace with " + str(len(targetspace)) + " categories, " +
str(testvectorantal) + " test items and " + str(trainvectorantal) +
" training cases. ", monitor)
confusion = ConfusionMatrix()
averagerankofauthorhit = 0
logger(
"Average linkage: " + str(averagelinkage) + " pool depth " +
str(itempooldepth), monitor)
for authorindex in testvectors:
logger(
str(authorindex) + "\t" +
str(facittable[authornametable[authorindex]]) + "===============",
debug)
targetscore = {}
for target in targets:
targetscore[target] = 0
for testfile in testvectors[authorindex]:
if averagelinkage: # take all test sentences and sum their scores
for target in targets:
print("Validation shape: {}".format(X_val.shape))
print("Training shape: {}".format(X_tr.shape))
eps = []
best_val_acc = 0
print "Start training\n"
for epoch in range(num_epochs):
# Calculate epoch time
start_time = time.time()
# Full pass training set
train_err = 0
train_batches = 0
confusion_train = ConfusionMatrix(n_class)
# Generate minibatches and train on each one of them
for batch in iterate_minibatches(X_tr, y_tr, mask_tr, batch_size, shuffle=True):
inputs, targets, in_masks = batch
tr_err, predict = train_fn(inputs, targets, in_masks)
train_err += tr_err
train_batches += 1
preds = np.argmax(predict, axis=-1)
confusion_train.batch_add(targets, preds)
train_loss = train_err / train_batches
train_accuracy = confusion_train.accuracy()
cf_train = confusion_train.ret_mat()
num_epochs = 25
train_acc= []
valid_acc = []
cur_loss = 0
loss = []
valid_loss = []
Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand")
Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand")
for epoch in range(num_epochs):
print epoch
cur_loss = 0
val_loss = 0
confusion_valid = ConfusionMatrix(classes)
confusion_train = ConfusionMatrix(classes)
B = np.array([])
Pos = np.empty((0, 1,3))
for im in Train:
Scan, Y_train,A, Post = DP.Sampling(im)
B = np.int32(np.append(B, Y_train))
random = np.arange(len(B))
print len(Pos)
Y_train = B[random]
Pos = A[random]
num_samples_train = Y_train.shape[0]
num_batches_train = num_samples_train // batch_size
for i in range(num_batches_train):
def run_blindset(topic, results_type, conf_matrix, blindset_name):
"""
Runs blindset test using credentials in ../Credentials.py
"""
# get credentials, import + export folders
import Credentials
active_adoption = Credentials.active_adoption
instance_creds = Credentials.ctx[active_adoption]
print(instance_creds)
print('print works')
workspace_id = Credentials.workspace_id[active_adoption][topic]
workspace_thresh = Credentials.calculate_workspace_thresh(topic)
conversation_version = Credentials.conversation_version
# import + export folders
import config
import time
data_folder = config.data_dir
export_folder = config.output_folder
timestr = time.strftime("%Y%m%d-%H%M")
blindset_name = blindset_name or topic + "_blindset.csv"
output_loc_results = os.path.join(
export_folder, "{}_results_raw_{}.csv".format(topic, timestr))
output_loc_metrics = os.path.join(
export_folder, "{}_results_metrics_{}.csv".format(topic, timestr))
output_loc_confmat = os.path.join(
export_folder, "{}_confmat_{}.png".format(topic, timestr))
# authenticate
if 'apikey' in instance_creds:
logger.debug("Authenticating (apikey)")
bs = blindset(apikey=instance_creds['apikey'],
url=instance_creds['url'],
threshold=workspace_thresh,
version=conversation_version)
elif 'password' in instance_creds:
logger.debug("Authenticating (username/password)")
bs = blindset(username=instance_creds['username'],
password=instance_creds['password'],
url=instance_creds['url'],
threshold=workspace_thresh,
version=conversation_version)
# run test
blindset_df = bs.import_blindset(os.path.join(data_folder, blindset_name))
# TODO: check blindset df
results = bs.run_blind_test(blindset_df, workspace_id)
# exports + metrics
if (results_type == 'raw') or (results_type == 'all'):
cols_export = [
col for col in results.columns.values if col != 'intent_correct'
]
results[cols_export].to_csv(output_loc_results, encoding='utf-8')
logger.info("Raw results exported to {}".format(output_loc_results))
if (results_type == 'metrics') or (results_type == 'all'):
met = Metrics(workspace_thresh)
metric_df, _ = met.get_all_metrics(results, detailed_results=True)
metric_df.to_csv(output_loc_metrics, encoding='utf-8')
logger.info(
"Metrics per intent exported to {}".format(output_loc_metrics))
# confusion matrix
if conf_matrix:
from confusionmatrix import ConfusionMatrix
cfn = ConfusionMatrix(workspace_thresh=workspace_thresh)
cfn.create(results, fig_path=output_loc_confmat)
#bs.plot_confusion_matrix(results, output_loc_confmat)
logger.info("Confusion matrix saved to {}".format(output_loc_confmat))
# print high-level metrics
overall_metrics = bs.calculate_overall_metrics(results,
av_method="weighted")
logger.info("Overall metrics for the workspace (weighted):")
logger.info(overall_metrics)
def runbatchtest(fraction, n: int = 100):
logger("{} {} {}".format(n, fraction, ticker), monitor)
keylist = list(vectorrepositoryall.keys())[:n]
random.shuffle(keylist)
split = int(len(keylist) * fraction)
train = keylist[:split]
test = keylist[split:]
logger("{} train vs {} test".format(len(train), len(test)), monitor)
ones = []
nils = []
dummymaxconfusionmatrix = ConfusionMatrix()
dummyrandomconfusionmatrix = ConfusionMatrix()
centroidconfusionmatrix = ConfusionMatrix()
poolconfusionmatrix = ConfusionMatrix()
for trainitem in test:
if illness[trainitem] == "1":
ones.append(vectorrepositoryall[trainitem])
else:
nils.append(vectorrepositoryall[trainitem])
onecentroid = sparsevectors.centroid(ones)
nilcentroid = sparsevectors.centroid(nils)
if len(nils) > len(ones):
dummymaxguess = "0"
else:
dummymaxguess = "1"
# factor = len(ones) / len(nils)
# no, bad idea, go for fifty-fifty
factor = 1 / 2
for testitem in test:
dummymaxconfusionmatrix.addconfusion(illness[testitem], dummymaxguess)
if random.random() > factor:
dummyrandomguess = "0"
else:
dummyrandomguess = "1"
dummyrandomconfusionmatrix.addconfusion(illness[testitem],
dummyrandomguess)
probe = vectorrepositoryall[testitem]
resultc = "0"
i1 = sparsevectors.sparsecosine(probe, onecentroid)
n1 = sparsevectors.sparsecosine(probe, nilcentroid)
if i1 > n1:
resultc = "1"
centroidconfusionmatrix.addconfusion(illness[testitem], resultc)
probeneighbours = {}
for targetitem in train:
probeneighbours[targetitem] = sparsevectors.sparsecosine(
probe, vectorrepositoryall[targetitem])
sortedfriends = sorted(probeneighbours,
key=lambda hh: probeneighbours[hh],
reverse=True)[:pooldepth]
illity = 0
result = "0"
for friend in sortedfriends:
if illness[friend] == "1":
illity += 1
if illity > pooldepth * factor:
result = "1"
nullity = pooldepth - illity
poolconfusionmatrix.addconfusion(illness[testitem], result)
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
testitem, illness[testitem], resultc, i1, n1, result, illity,
nullity, pooldepth))
print("RANDOM ----------------")
dummyrandomconfusionmatrix.evaluate()
print("MAX ----------------")
dummymaxconfusionmatrix.evaluate()
print("CENTROID ----------------")
centroidconfusionmatrix.evaluate()
print("NEIGHBOURS --------------")
poolconfusionmatrix.evaluate()
def run_kfold(topic, no_folds, results_type, conf_matrix):
"""
Runs kfold test using credentials in ../Credentials.py
"""
# get credentials, import + export folders
import Credentials
active_adoption = Credentials.active_adoption
instance_creds = Credentials.ctx[active_adoption]
workspace_id = Credentials.workspace_id[active_adoption][topic]
workspace_thresh = Credentials.calculate_workspace_thresh(topic)
conversation_version = Credentials.conversation_version
# import + export folders
import config
import time
data_folder = config.data_dir
export_folder = config.output_folder
timestr = time.strftime("%Y%m%d-%H%M")
output_loc_results = os.path.join(
export_folder, "{}_kfold_results_raw_{}.csv".format(topic, timestr))
output_loc_metrics = os.path.join(
export_folder, "{}_kfold_results_metrics_{}.csv".format(topic, timestr))
output_loc_confmat = os.path.join(
export_folder, "{}_kfold_confmat_{}.png".format(topic, timestr))
# authenticate
if 'apikey' in instance_creds:
logger.debug("Authenticating (apikey)")
kf = kfoldtest(n_folds=no_folds, apikey=instance_creds['apikey'],
url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version)
elif 'password' in instance_creds:
logger.debug("Authenticating (username/password)")
kf = kfoldtest(n_folds=no_folds, username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh,
version=conversation_version)
# get train df from watson + check there are sufficient workspaces to run the test
train_df = kf.intent_df_from_watson(workspace_id)
kf.check_sufficient_workspaces()
# create folds in WA if above is true
folds = kf.create_folds(method='kfold')
kf.create_kfold_WA(folds)
available_flag = False
while available_flag == False:
logger.info("Checking workspaces..")
available_flag = kf.check_workspaces_status()
time.sleep(20)
# run kfold test
try:
results = kf.run_kfold_test(folds)
if (results_type == 'raw') or (results_type == 'all'):
results.to_csv(output_loc_results)
classification_report = kf.create_classification_report(results)
if (results_type == 'metrics') or (results_type == 'all'):
metrics = Metrics(workspace_thresh)
metric_df = metrics.get_all_metrics_CV(
results, fold_col='fold', detailed_results=False)
metric_df.to_csv(output_loc_metrics)
# TODO: confusion matrix
if conf_matrix:
from confusionmatrix import ConfusionMatrix
cfn = ConfusionMatrix(workspace_thresh=workspace_thresh)
cfn.create(results, fig_path=output_loc_confmat)
logger.info("Confusion matrix saved to {}".format(
output_loc_confmat))
finally:
# regardless of what happens above, delete the temporary workspaces before exiting
kf.delete_kfold_workspaces()
for item in testers:
neighbours[item] = {}
for otheritem in targetspace.items():
if testitemspace.name[item] == targetspace.name[otheritem]:
continue
neighbours[item][otheritem] = sparsevectors.sparsecosine(
testitemspace.indexspace[item],
targetspace.indexspace[otheritem])
logger("Done calculating neighbours", monitor)
logger("Pool depth " + str(itempooldepth), monitor)
if averagelinkage:
logger("Averagelinkage", monitor)
if votelinkage:
logger("Votelinkage", monitor)
confusion = ConfusionMatrix()
primeconfusion = ConfusionMatrix()
targetscore = {}
for item in testers:
sortedneighbours = sorted(neighbours[item],
key=lambda hh: neighbours[item][hh],
reverse=True)[:itempooldepth]
primeconfusion.addconfusion(facittable[testitemspace.name[item]],
targetspace.category[sortedneighbours[0]])
for target in categories:
targetscore[target] = 0
if averagelinkage: # take all test neighbours and sum their scores
for neighbour in sortedneighbours:
targetscore[targetspace.
category[neighbour]] += neighbours[item][neighbour]
elif votelinkage:
print("\n")
for item in items:
print(str(item), itemspace.name[item])
result = {}
prunedresult = {}
for c in categories:
result[c] = {}
prunedresult[c] = {}
logger("Pool depth " + str(itempooldepth), monitor)
if averagelinkage:
logger("Averagelinkage", monitor)
if votelinkage:
logger("Votelinkage", monitor)
confusion = ConfusionMatrix()
prunedconfusion = ConfusionMatrix()
targetscore = {}
prunedtargetscore = {}
for item in testvectors:
sortedneighbours = sorted(neighbours[item],
key=lambda hh: neighbours[item][hh],
reverse=True)[:itempooldepth]
if cleanup:
prunedsortedneighbours = sorted(
prunedneighbours[item],
key=lambda hh: prunedneighbours[item][hh],
reverse=True)[:itempooldepth]
for target in categories:
targetscore[target] = 0
prunedtargetscore[target] = 0
num_epochs = 175
train_acc= []
valid_acc = []
cur_loss = 0
loss = []
valid_loss = []
Train = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand")
Test = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand")
import gc
for epoch in range(num_epochs):
cur_loss = 0
val_loss = 0
confusion_valid = ConfusionMatrix(2)
confusion_train = ConfusionMatrix(2)
for im in Train:
XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS)
num_samples_train = Y_train.shape[0]
num_batches_train = num_samples_train // batch_size
for i in range(num_batches_train):
idx = range(i*batch_size, (i+1)*batch_size)
xy_batch = XY[idx]
xz_batch = XZ[idx]
yz_batch = YZ[idx]
target_batch = Y_train[idx]
batch_loss = f_train(xy_batch, xz_batch, yz_batch ,target_batch) #this will do the backprop pass
def main():
parser = argparse.ArgumentParser(description='Memlearn Chainer ver')
parser.add_argument('--batchsize', '-b', type=int, default=100,
help='Number of images in each mini-batch')
parser.add_argument('--epoch', '-e', type=int, default=20,
help='Number of sweeps over the dataset to train')
parser.add_argument('--frequency', '-f', type=int, default=-1,
help='Frequency of taking a snapshot')
parser.add_argument('--gpu', '-g', type=int, default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out', '-o', default='result',
help='Directory to output the result')
parser.add_argument('--resume', '-r', default='',
help='Resume the training from snapshot')
parser.add_argument('--train', '-i', nargs='*',
help='Train data (all except test by default)')
parser.add_argument('--test', '-t', nargs='*',
help='Test data (you have to specify at least one')
parser.add_argument('--sliceb', '-s', type=int, default=1,
help='Time slice block')
parser.add_argument('--list_dataset', action='store_true')
parser.add_argument('--dataset', default="/data/data_filtered.pickle")
args = parser.parse_args()
if args.list_dataset:
with open(args.dataset, 'rb') as fp:
dataset = pickle.load(fp)
print('\n'.join(['{} {}'.format(n, l) for n, l in dataset]))
raise SystemExit
if not args.test:
raise SystemExit("you have to specify at least one test, see --list_dataset for available datasets")
print('GPU: {}'.format(args.gpu))
print('# Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('')
# Set up a neural network to train
# Classifier reports softmax cross entropy loss and accuracy at every
# iteration, which will be used by the PrintReport extension below.
model = L.Classifier(VGG(2))
#model = L.Classifier(SVM())
if args.gpu >= 0:
# Make a specified GPU current
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu() # Copy the model to the GPU
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# Load the MEMORY dataset
train, test = prep_windowed_datasets(args.dataset, args.test, args.train, slice_merge=args.sliceb)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test, args.batchsize,
repeat=False, shuffle=False)
# Set up a trainer
updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
# Evaluate the model with the test dataset for each epoch
#trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu))
trainer.extend(ConfusionMatrix(test_iter, model, device=args.gpu))
# Dump a computational graph from 'loss' variable at the first iteration
# The "main" refers to the target link of the "main" optimizer.
trainer.extend(extensions.dump_graph('main/loss'))
# Take a snapshot for each specified epoch
frequency = args.epoch if args.frequency == -1 else max(1, args.frequency)
trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch'))
# Write a log of evaluation statistics for each epoch
trainer.extend(extensions.LogReport())
# Save two plot images to the result dir
if extensions.PlotReport.available():
trainer.extend(
extensions.PlotReport(['main/loss', 'validation/main/loss'],
'epoch', file_name='loss.png'))
trainer.extend(
extensions.PlotReport(
['main/accuracy', 'validation/main/accuracy'],
'epoch', file_name='accuracy.png'))
# Print selected entries of the log to stdout
# Here "main" refers to the target link of the "main" optimizer again, and
# "validation" refers to the default name of the Evaluator extension.
# Entries other than 'epoch' are reported by the Classifier link, called by
# either the updater or the evaluator.
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'elapsed_time']))
# Print a progress bar to stdout
trainer.extend(extensions.ProgressBar())
if args.resume:
# Resume from a snapshot
chainer.serializers.load_npz(args.resume, trainer)
# Run the training
trainer.run()
y_test,
writer=dev_summary_writer)
if accuracy > max_acc:
max_acc = accuracy
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
else:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
saver.restore(sess, ckpt.model_checkpoint_path)
accuracy, predictions = dev_step(x_test, y_test)
#dev_step(x_dev, y_dev, writer=dev_summary_writer)
outputs = np.argmax(predictions, axis=-1)
print(outputs.shape)
confusion_test = ConfusionMatrix(num_classes)
y_test = np.argmax(y_test, axis=-1)
print(y_test.shape)
confusion_test.batch_add(y_test, outputs)
test_accuracy = confusion_test.accuracy()
a, positive_predictive_value = confusion_test.positive_predictive_value()
b, negative_predictive_value = confusion_test.negative_predictive_value()
e, F1 = confusion_test.F1()
f, MCC = confusion_test.matthews_correlation()
cf_val = confusion_test.ret_mat()
print("FINAL TEST RESULTS")
print(confusion_test)
print(cf_val)
})
d_acc = sess.run(model.domain_acc,
feed_dict={
model.input_x: combined_test_imgs,
model.domain: combined_test_domain,
model.dropout_keep_prob: 1,
model.training: False
})
print('\nDomain adaptation training')
print('Source (mouse) accuracy:', source_acc)
print('target (human) accuracy:', target_acc)
print('Domain accuracy:', d_acc)
outputs = np.argmax(predictions, axis=-1)
print(outputs.shape)
confusion_test = ConfusionMatrix(10)
y_test = np.argmax(y_human_test, axis=-1)
print(y_test.shape)
confusion_test.batch_add(y_test, outputs)
test_accuracy = confusion_test.accuracy()
positive_predictive_value = confusion_test.positive_predictive_value(
)
negative_predictive_value = confusion_test.negative_predictive_value(
)
F1 = confusion_test.F1()
MCC = confusion_test.matthews_correlation()
cf_val = confusion_test.ret_mat()
print("FINAL TEST RESULTS")
