def evaluate(dataset, classifier, groundTruth):
'''evaluate the classifier on the given dataset, using only the points
that are in the groundTruth. It then returns the confusion matrix.
- dataset is the dataset from which to get the points
- classifier is a function which given a point returns its class
- groundTruth is a map from the points to classify to their respective class
'''
total = len(groundTruth)
done = 0
confusion = ConfusionMatrix()
for (pointId, expected) in groundTruth.items():
try:
found = classifier(dataset.point(pointId))
confusion.add(expected, found, pointId)
except Exception, e:
print 'WARNING: Could not classify point', pointId, 'because', str(
e)
done += 1
print '\r[%d%%]' % int(done * 100 / total),
def matrix(self):
"""Creates a Contingenz Matrix using ConfusionMatrix of NLTK"""
cm = ConfusionMatrix(self.reference_tags, self.test_tags) # first reference, then test!
#f = codecs.open("C:\\Users\\"+self.user+"\\Downloads\\continenzmatrix.txt", "w", "utf-8")
f = codecs.open("Results\\contingenzmatrix.txt","w","utf-8")
f.write(cm.pp())
f.close()
###print contingenzliste
#g = codecs.open("C:\\Users\\"+self.user+"\\Downloads\\contingenzliste.txt", "w", "utf-8")
#g.writelines(self.contingenzliste)
#g.close()
values_not_null = cm.get_values_not_null()
return values_not_null
def __init__(self, num_classes, normalized=False, ignore_index=None):
super(IoU, self).__init__()
self.conf_metric = ConfusionMatrix(num_classes, normalized)
if ignore_index is None:
self.ignore_index = None
elif isinstance(ignore_index, int):
self.ignore_index = (ignore_index, )
else:
try:
self.ignore_index = tuple(ignore_index)
except TypeError:
raise ValueError("'ignore_index' must be an int or iterable")
def evaluate(classifier, dataset, groundTruth, confusion=None, verbose=True):
"""Evaluate the classifier on the given dataset and returns the confusion matrix.
Uses only the points that are in the groundTruth parameter for the evaluation.
Parameters
----------
classifier : a function which given a point returns its class
dataset : the dataset from which to get the points
groundTruth : a map from the points to classify to their respective class
"""
progress = TextProgress(len(groundTruth))
done = 0
confusion = confusion or ConfusionMatrix()
for pointId, expected in groundTruth.items():
try:
found = classifier(dataset.point(pointId))
confusion.add(expected, found, pointId)
except Exception as e:
log.warning('Could not classify point "%s" because %s' %
(pointId, str(e)))
raise
done += 1
if verbose: progress.update(done)
return confusion
def trial(trial_number, metric, K, N, distribution, xi):
X = point.PointCloud.generate_with_classes(N, K, distribution, [True, False])
Y = point.PointCloud.generate(N, M, distribution)
for y in Y:
closest_point = y.nearest_point(X, metric)
y.clazz = closest_point.clazz
perturbed_Y = deepcopy(Y).perturb_points(xi)
for y in perturbed_Y:
closest_point = y.nearest_point(X, metric)
y.clazz = closest_point.clazz
confusion_matrix = ConfusionMatrix.generate_from_point_clouds(Y, perturbed_Y)
result = [trial_number, metric, K, N, xi.sigma] + confusion_matrix.row()
print(result)
return result
def evaluate(dataset, classifier, groundTruth):
'''evaluate the classifier on the given dataset, using only the points
that are in the groundTruth. It then returns the confusion matrix.
- dataset is the dataset from which to get the points
- classifier is a function which given a point returns its class
- groundTruth is a map from the points to classify to their respective class
'''
total = len(groundTruth)
done = 0
confusion = ConfusionMatrix()
for (pointId, expected) in groundTruth.items():
try:
found = classifier(dataset.point(pointId))
confusion.add(expected, found, pointId)
except Exception, e:
print 'WARNING: Could not classify point', pointId, 'because', str(e)
done += 1
print '\r[%d%%]' % int(done*100 / total),
metric = random.choice(metrics)
return N, K, distribution, xi, metric
def trial(trial_number, metric, K, N, distribution, xi):
X = point.PointCloud.generate_with_classes(N, K, distribution, [True, False])
Y = point.PointCloud.generate(N, M, distribution)
for y in Y:
closest_point = y.nearest_point(X, metric)
y.clazz = closest_point.clazz
perturbed_Y = deepcopy(Y).perturb_points(xi)
for y in perturbed_Y:
closest_point = y.nearest_point(X, metric)
y.clazz = closest_point.clazz
confusion_matrix = ConfusionMatrix.generate_from_point_clouds(Y, perturbed_Y)
result = [trial_number, metric, K, N, xi.sigma] + confusion_matrix.row()
print(result)
return result
results = []
for trial_number in range(trials):
N, K, distribution, xi, metric = generate_params(N_Range, K_Range, metrics, distributions, sigma_range)
results.append(trial(trial_number, metric, K, N, distribution, xi))
tablemaker.make_table('experiment1-2(1)', ['Trial', 'Metric', 'K', 'N', 'sigma'] + ConfusionMatrix.headers(), results)
all_mon = train_out[3:]
grd_mon = train_out[:len(all_grads)]
upd_mon = train_out[len(all_grads):]
for pm, gm, um in zip(trainable_params, grd_mon, upd_mon):
if '.b' not in pm.name:
pad = (40-len(pm.name))*" "
print "%s \t %.5e \t %.5e \t %.5e" % (
pm.name + pad,
np.linalg.norm(gm),
np.linalg.norm(um),
np.linalg.norm(pm.get_value())
)
cost_train_lst += [cost_train]
conf_train = ConfusionMatrix(num_classes)
for i in range(x_train.shape[0] // 1000):
probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000])
preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1)
conf_train.batch_add(
y_train[i*1000:(i+1)*1000].flatten(),
preds_train_flat
)
if last_decay > args.decayinterval and epoch > args.nodecay:
last_decay = 0
old_lr = sh_lr.get_value(sh_lr)
new_lr = old_lr / args.decayfac
sh_lr.set_value(lasagne.utils.floatX(new_lr))
print "Decay lr from %f to %f" % (float(old_lr), float(new_lr))
else:
targetspace[tv[0]], tv[1])
logger("Done training files.", monitor)
if outputmodel:
# output character patterns to be able to generate new tweetvectors for separate testing on trained data
stringspace.saveelementspace(charactervectorspacefilename)
# output model here with info about the category of each model item
with open(categorymodelfilename, "wb") as outfile:
pickle.dump(targetspace, outfile)
logger(
"Testing targetspace with " + str(len(targetspace)) + " categories, " +
str(testvectorantal) + " test items and " + str(trainvectorantal) +
" training cases. ", monitor)
confusion = ConfusionMatrix()
averagerankofauthorhit = 0
logger(
"Average linkage: " + str(averagelinkage) + " pool depth " +
str(itempooldepth), monitor)
for authorindex in testvectors:
logger(
str(authorindex) + "\t" +
str(facittable[authornametable[authorindex]]) + "===============",
debug)
targetscore = {}
for target in targets:
targetscore[target] = 0
for testfile in testvectors[authorindex]:
if averagelinkage: # take all test sentences and sum their scores
for target in targets:
print("Validation shape: {}".format(X_val.shape))
print("Training shape: {}".format(X_tr.shape))
eps = []
best_val_acc = 0
print "Start training\n"
for epoch in range(num_epochs):
# Calculate epoch time
start_time = time.time()
# Full pass training set
train_err = 0
train_batches = 0
confusion_train = ConfusionMatrix(n_class)
# Generate minibatches and train on each one of them
for batch in iterate_minibatches(X_tr, y_tr, mask_tr, batch_size, shuffle=True):
inputs, targets, in_masks = batch
tr_err, predict = train_fn(inputs, targets, in_masks)
train_err += tr_err
train_batches += 1
preds = np.argmax(predict, axis=-1)
confusion_train.batch_add(targets, preds)
train_loss = train_err / train_batches
train_accuracy = confusion_train.accuracy()
cf_train = confusion_train.ret_mat()
def main():
parser = argparse.ArgumentParser(description='Memlearn Chainer ver')
parser.add_argument('--batchsize', '-b', type=int, default=100,
help='Number of images in each mini-batch')
parser.add_argument('--epoch', '-e', type=int, default=20,
help='Number of sweeps over the dataset to train')
parser.add_argument('--frequency', '-f', type=int, default=-1,
help='Frequency of taking a snapshot')
parser.add_argument('--gpu', '-g', type=int, default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out', '-o', default='result',
help='Directory to output the result')
parser.add_argument('--resume', '-r', default='',
help='Resume the training from snapshot')
parser.add_argument('--train', '-i', nargs='*',
help='Train data (all except test by default)')
parser.add_argument('--test', '-t', nargs='*',
help='Test data (you have to specify at least one')
parser.add_argument('--sliceb', '-s', type=int, default=1,
help='Time slice block')
parser.add_argument('--list_dataset', action='store_true')
parser.add_argument('--dataset', default="/data/data_filtered.pickle")
args = parser.parse_args()
if args.list_dataset:
with open(args.dataset, 'rb') as fp:
dataset = pickle.load(fp)
print('\n'.join(['{} {}'.format(n, l) for n, l in dataset]))
raise SystemExit
if not args.test:
raise SystemExit("you have to specify at least one test, see --list_dataset for available datasets")
print('GPU: {}'.format(args.gpu))
print('# Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('')
# Set up a neural network to train
# Classifier reports softmax cross entropy loss and accuracy at every
# iteration, which will be used by the PrintReport extension below.
model = L.Classifier(VGG(2))
#model = L.Classifier(SVM())
if args.gpu >= 0:
# Make a specified GPU current
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu() # Copy the model to the GPU
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# Load the MEMORY dataset
train, test = prep_windowed_datasets(args.dataset, args.test, args.train, slice_merge=args.sliceb)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test, args.batchsize,
repeat=False, shuffle=False)
# Set up a trainer
updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
# Evaluate the model with the test dataset for each epoch
#trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu))
trainer.extend(ConfusionMatrix(test_iter, model, device=args.gpu))
# Dump a computational graph from 'loss' variable at the first iteration
# The "main" refers to the target link of the "main" optimizer.
trainer.extend(extensions.dump_graph('main/loss'))
# Take a snapshot for each specified epoch
frequency = args.epoch if args.frequency == -1 else max(1, args.frequency)
trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch'))
# Write a log of evaluation statistics for each epoch
trainer.extend(extensions.LogReport())
# Save two plot images to the result dir
if extensions.PlotReport.available():
trainer.extend(
extensions.PlotReport(['main/loss', 'validation/main/loss'],
'epoch', file_name='loss.png'))
trainer.extend(
extensions.PlotReport(
['main/accuracy', 'validation/main/accuracy'],
'epoch', file_name='accuracy.png'))
# Print selected entries of the log to stdout
# Here "main" refers to the target link of the "main" optimizer again, and
# "validation" refers to the default name of the Evaluator extension.
# Entries other than 'epoch' are reported by the Classifier link, called by
# either the updater or the evaluator.
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'elapsed_time']))
# Print a progress bar to stdout
trainer.extend(extensions.ProgressBar())
if args.resume:
# Resume from a snapshot
chainer.serializers.load_npz(args.resume, trainer)
# Run the training
trainer.run()
#print("The batch loss is : {} for itteration number : {}".format(res[1],i))
#embed()######new test
#pred_network= sess.run(fetches=[y], feed_dict={x_pl: x_batch,is_training: False})
#pred=np.asarray(pred_network)
#pred=pred[0][:,1]
#fpr, tpr, thresholds = metrics.roc_curve(target_batch, pred, pos_label=0)
#aucScore=metrics.auc(fpr, tpr)
#aucList.append(aucScore)
#print("The auc score is {} ".format(aucScore))
#accuracyList.append(acc)
batch_loss = res[1] #this will do the complete backprob pass
cur_loss += batch_loss
loss += [cur_loss / batch_size]
confusion_valid = ConfusionMatrix(num_classes)
confusion_train = ConfusionMatrix(num_classes)
print("Training done")
trainLoss = []
trainAcc = []
trainAuc = []
for i in range(num_batches_train): # The model is evaluated
index = range(i * batch_size, (i + 1) * batch_size)
x_batch = x_train[index]
targets_batch = targets_train[index]
fetches_train = [cross_entropy]
feed_dict_train = {
x_pl: x_batch,
y_: myOneHot(targets_batch),
is_training: False
weight = Weighting(new_cleaned_data, new_terms)
tfidf = weight.get_tf_idf_weighting()
idf = weight.get_idf()
nb = NBMultinomial()
nb.fit(new_cleaned_data, new_terms, data_train[i]["target"], stopwords,
idf, tfidf)
for j in range(len(data_test[i]["tweet"])):
print("Test ke " + str(j))
prediction = nb.predict(data_test[i]["tweet"][j],
data_test[i]["target"][j])
y_test.append(data_test[i]["target"][j])
y_pred.append(prediction)
cm = ConfusionMatrix()
accuracy, accuracy_each_class, precision_each_class, recall_each_class, fmeasure_each_class = cm.score(
y_test, y_pred)
acc_neg.append(accuracy_each_class[0])
acc_net.append(accuracy_each_class[1])
acc_pos.append(accuracy_each_class[2])
prec_neg.append(precision_each_class[0])
prec_net.append(precision_each_class[1])
prec_pos.append(precision_each_class[2])
recall_neg.append(recall_each_class[0])
recall_net.append(recall_each_class[1])
recall_pos.append(recall_each_class[2])
fmeasure_neg.append(fmeasure_each_class[0])
fmeasure_net.append(fmeasure_each_class[1])
fmeasure_pos.append(fmeasure_each_class[2])
num_epochs = 25
train_acc = []
valid_acc = []
cur_loss = 0
loss = []
valid_loss = []
threshold, upper, lower = 0.5, 1, 0
Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand")
Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand")
import gc
for epoch in range(num_epochs):
cur_loss = 0
val_loss = 0
confusion_valid = ConfusionMatrix(2)
confusion_train = ConfusionMatrix(2)
for im in Train:
XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS)
num_samples_train = Y_train.shape[0]
num_batches_train = num_samples_train // batch_size
for i in range(num_batches_train):
idx = range(i * batch_size, (i + 1) * batch_size)
xy_batch = XY[idx]
xz_batch = XZ[idx]
yz_batch = YZ[idx]
target_batch = np.float32(Y_train[idx].reshape(batch_size, 1))
batch_loss = f_train(xy_batch, xz_batch, yz_batch,
target_batch) #this will do the backprop pass
cur_loss += batch_loss[0] / batch_size
ffn = FeedforwardNetwork()
ffn.add(LinearLayer(num_inputs, num_hidden_units))
ffn.add(ReluActivationLayer())
ffn.add(LinearLayer(num_hidden_units, num_classes))
ffn.add(SoftmaxActivationLayer())
losslayer = CrossEntropyLoss()
print "Network"
print ffn
print "Loss", losslayer
print ""
acc = []
for epoch in range(num_epochs):
confusion = ConfusionMatrix(num_classes)
for i in range(num_batches):
idx = range(i*batch_size, (i+1)*batch_size)
x_batch = x_train[idx]
target_batch = targets_train[idx]
y_probs = ffn.forward(x_batch)
loss = losslayer.fprop(y_probs, target_batch)
delta = losslayer.bprop(y_probs, target_batch)
ffn.backward(delta)
ffn.update(learning_rate)
confusion.batch_add(target_batch.argmax(-1), y_probs.argmax(-1))
curr_acc = confusion.accuracy()
print "Epoch %i : Loss %f Train acc %f" % (epoch, loss, curr_acc)
acc += [curr_acc]
print("\nEvaluation:")
accuracy, predictions = dev_step(x_test,
y_test,
writer=dev_summary_writer)
if accuracy > max_acc:
max_acc = accuracy
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
else:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
saver.restore(sess, ckpt.model_checkpoint_path)
accuracy, predictions = dev_step(
x_test,
y_test) # dev_step(x_dev, y_dev, writer=dev_summary_writer)
outputs = np.argmax(predictions, axis=-1)
print(outputs.shape)
confusion_test = ConfusionMatrix(num_classes)
y_test = np.argmax(y_test, axis=-1)
print(y_test.shape)
confusion_test.batch_add(y_test, outputs)
test_accuracy = confusion_test.accuracy()
cf_test = confusion_test.ret_mat()
print("FINAL TEST RESULTS")
print(confusion_test)
print(" test accuracy:\t\t{:.2f} %".format(test_accuracy * 100))
print(" test Gorodkin:\t\t{:.2f}".format(gorodkin(cf_test)))
print(" test IC:\t\t{:.2f}".format(IC(cf_test)))
def runbatchtest(fraction, n: int = 100):
logger("{} {} {}".format(n, fraction, ticker), monitor)
keylist = list(vectorrepositoryall.keys())[:n]
random.shuffle(keylist)
split = int(len(keylist) * fraction)
train = keylist[:split]
test = keylist[split:]
logger("{} train vs {} test".format(len(train), len(test)), monitor)
ones = []
nils = []
dummymaxconfusionmatrix = ConfusionMatrix()
dummyrandomconfusionmatrix = ConfusionMatrix()
centroidconfusionmatrix = ConfusionMatrix()
poolconfusionmatrix = ConfusionMatrix()
for trainitem in test:
if illness[trainitem] == "1":
ones.append(vectorrepositoryall[trainitem])
else:
nils.append(vectorrepositoryall[trainitem])
onecentroid = sparsevectors.centroid(ones)
nilcentroid = sparsevectors.centroid(nils)
if len(nils) > len(ones):
dummymaxguess = "0"
else:
dummymaxguess = "1"
# factor = len(ones) / len(nils)
# no, bad idea, go for fifty-fifty
factor = 1 / 2
for testitem in test:
dummymaxconfusionmatrix.addconfusion(illness[testitem], dummymaxguess)
if random.random() > factor:
dummyrandomguess = "0"
else:
dummyrandomguess = "1"
dummyrandomconfusionmatrix.addconfusion(illness[testitem],
dummyrandomguess)
probe = vectorrepositoryall[testitem]
resultc = "0"
i1 = sparsevectors.sparsecosine(probe, onecentroid)
n1 = sparsevectors.sparsecosine(probe, nilcentroid)
if i1 > n1:
resultc = "1"
centroidconfusionmatrix.addconfusion(illness[testitem], resultc)
probeneighbours = {}
for targetitem in train:
probeneighbours[targetitem] = sparsevectors.sparsecosine(
probe, vectorrepositoryall[targetitem])
sortedfriends = sorted(probeneighbours,
key=lambda hh: probeneighbours[hh],
reverse=True)[:pooldepth]
illity = 0
result = "0"
for friend in sortedfriends:
if illness[friend] == "1":
illity += 1
if illity > pooldepth * factor:
result = "1"
nullity = pooldepth - illity
poolconfusionmatrix.addconfusion(illness[testitem], result)
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
testitem, illness[testitem], resultc, i1, n1, result, illity,
nullity, pooldepth))
print("RANDOM ----------------")
dummyrandomconfusionmatrix.evaluate()
print("MAX ----------------")
dummymaxconfusionmatrix.evaluate()
print("CENTROID ----------------")
centroidconfusionmatrix.evaluate()
print("NEIGHBOURS --------------")
poolconfusionmatrix.evaluate()
def run_kfold(topic, no_folds, results_type, conf_matrix):
"""
Runs kfold test using credentials in ../Credentials.py
"""
# get credentials, import + export folders
import Credentials
active_adoption = Credentials.active_adoption
instance_creds = Credentials.ctx[active_adoption]
workspace_id = Credentials.workspace_id[active_adoption][topic]
workspace_thresh = Credentials.calculate_workspace_thresh(topic)
conversation_version = Credentials.conversation_version
# import + export folders
import config
import time
data_folder = config.data_dir
export_folder = config.output_folder
timestr = time.strftime("%Y%m%d-%H%M")
output_loc_results = os.path.join(
export_folder, "{}_kfold_results_raw_{}.csv".format(topic, timestr))
output_loc_metrics = os.path.join(
export_folder, "{}_kfold_results_metrics_{}.csv".format(topic, timestr))
output_loc_confmat = os.path.join(
export_folder, "{}_kfold_confmat_{}.png".format(topic, timestr))
# authenticate
if 'apikey' in instance_creds:
logger.debug("Authenticating (apikey)")
kf = kfoldtest(n_folds=no_folds, apikey=instance_creds['apikey'],
url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version)
elif 'password' in instance_creds:
logger.debug("Authenticating (username/password)")
kf = kfoldtest(n_folds=no_folds, username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh,
version=conversation_version)
# get train df from watson + check there are sufficient workspaces to run the test
train_df = kf.intent_df_from_watson(workspace_id)
kf.check_sufficient_workspaces()
# create folds in WA if above is true
folds = kf.create_folds(method='kfold')
kf.create_kfold_WA(folds)
available_flag = False
while available_flag == False:
logger.info("Checking workspaces..")
available_flag = kf.check_workspaces_status()
time.sleep(20)
# run kfold test
try:
results = kf.run_kfold_test(folds)
if (results_type == 'raw') or (results_type == 'all'):
results.to_csv(output_loc_results)
classification_report = kf.create_classification_report(results)
if (results_type == 'metrics') or (results_type == 'all'):
metrics = Metrics(workspace_thresh)
metric_df = metrics.get_all_metrics_CV(
results, fold_col='fold', detailed_results=False)
metric_df.to_csv(output_loc_metrics)
# TODO: confusion matrix
if conf_matrix:
from confusionmatrix import ConfusionMatrix
cfn = ConfusionMatrix(workspace_thresh=workspace_thresh)
cfn.create(results, fig_path=output_loc_confmat)
logger.info("Confusion matrix saved to {}".format(
output_loc_confmat))
finally:
# regardless of what happens above, delete the temporary workspaces before exiting
kf.delete_kfold_workspaces()
for item in testers:
neighbours[item] = {}
for otheritem in targetspace.items():
if testitemspace.name[item] == targetspace.name[otheritem]:
continue
neighbours[item][otheritem] = sparsevectors.sparsecosine(
testitemspace.indexspace[item],
targetspace.indexspace[otheritem])
logger("Done calculating neighbours", monitor)
logger("Pool depth " + str(itempooldepth), monitor)
if averagelinkage:
logger("Averagelinkage", monitor)
if votelinkage:
logger("Votelinkage", monitor)
confusion = ConfusionMatrix()
primeconfusion = ConfusionMatrix()
targetscore = {}
for item in testers:
sortedneighbours = sorted(neighbours[item],
key=lambda hh: neighbours[item][hh],
reverse=True)[:itempooldepth]
primeconfusion.addconfusion(facittable[testitemspace.name[item]],
targetspace.category[sortedneighbours[0]])
for target in categories:
targetscore[target] = 0
if averagelinkage: # take all test neighbours and sum their scores
for neighbour in sortedneighbours:
targetscore[targetspace.
category[neighbour]] += neighbours[item][neighbour]
elif votelinkage:
num_epochs = 175
train_acc= []
valid_acc = []
cur_loss = 0
loss = []
valid_loss = []
Train = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand")
Test = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand")
import gc
for epoch in range(num_epochs):
cur_loss = 0
val_loss = 0
confusion_valid = ConfusionMatrix(2)
confusion_train = ConfusionMatrix(2)
for im in Train:
XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS)
num_samples_train = Y_train.shape[0]
num_batches_train = num_samples_train // batch_size
for i in range(num_batches_train):
idx = range(i*batch_size, (i+1)*batch_size)
xy_batch = XY[idx]
xz_batch = XZ[idx]
yz_batch = YZ[idx]
target_batch = Y_train[idx]
batch_loss = f_train(xy_batch, xz_batch, yz_batch ,target_batch) #this will do the backprop pass
all_mon = train_out[3:]
grd_mon = train_out[:len(all_grads)]
upd_mon = train_out[len(all_grads):]
for pm, gm, um in zip(trainable_params, grd_mon, upd_mon):
if '.b' not in pm.name:
pad = (40-len(pm.name))*" "
print("%s \t %.5e \t %.5e \t %.5e" % (
pm.name + pad,
np.linalg.norm(gm),
np.linalg.norm(um),
np.linalg.norm(pm.get_value())
))
cost_train_lst += [cost_train]
conf_train = ConfusionMatrix(num_classes)
for i in range(x_train.shape[0] // 1000):
probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000])
preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1)
conf_train.batch_add(
y_train[i*1000:(i+1)*1000].flatten(),
preds_train_flat
)
if last_decay > args.decayinterval and epoch > args.nodecay:
last_decay = 0
old_lr = sh_lr.get_value(sh_lr)
new_lr = old_lr / args.decayfac
sh_lr.set_value(lasagne.utils.floatX(new_lr))
print("Decay lr from %f to %f" % (float(old_lr), float(new_lr)))
else:
print("\n")
for item in items:
print(str(item), itemspace.name[item])
result = {}
prunedresult = {}
for c in categories:
result[c] = {}
prunedresult[c] = {}
logger("Pool depth " + str(itempooldepth), monitor)
if averagelinkage:
logger("Averagelinkage", monitor)
if votelinkage:
logger("Votelinkage", monitor)
confusion = ConfusionMatrix()
prunedconfusion = ConfusionMatrix()
targetscore = {}
prunedtargetscore = {}
for item in testvectors:
sortedneighbours = sorted(neighbours[item],
key=lambda hh: neighbours[item][hh],
reverse=True)[:itempooldepth]
if cleanup:
prunedsortedneighbours = sorted(
prunedneighbours[item],
key=lambda hh: prunedneighbours[item][hh],
reverse=True)[:itempooldepth]
for target in categories:
targetscore[target] = 0
prunedtargetscore[target] = 0
class IoU(metric.Metric):
"""Computes the intersection over union (IoU) per class and corresponding
mean (mIoU).
Intersection over union (IoU) is a common evaluation metric for semantic
segmentation. The predictions are first accumulated in a confusion matrix
and the IoU is computed from it as follows:
IoU = true_positive / (true_positive + false_positive + false_negative).
Keyword arguments:
- num_classes (int): number of classes in the classification problem
- normalized (boolean, optional): Determines whether or not the confusion
matrix is normalized or not. Default: False.
- ignore_index (int or iterable, optional): Index of the classes to ignore
when computing the IoU. Can be an int, or any iterable of ints.
"""
def __init__(self, num_classes, normalized=False, ignore_index=None):
super(IoU, self).__init__()
self.conf_metric = ConfusionMatrix(num_classes, normalized)
if ignore_index is None:
self.ignore_index = None
elif isinstance(ignore_index, int):
self.ignore_index = (ignore_index, )
else:
try:
self.ignore_index = tuple(ignore_index)
except TypeError:
raise ValueError("'ignore_index' must be an int or iterable")
def reset(self):
self.conf_metric.reset()
def add(self, predicted, target):
"""Adds the predicted and target pair to the IoU metric.
Keyword arguments:
- predicted (Tensor): Can be a (N, K, H, W) tensor of
predicted scores obtained from the model for N examples and K classes,
or (N, H, W) tensor of integer values between 0 and K-1.
- target (Tensor): Can be a (N, K, H, W) tensor of
target scores for N examples and K classes, or (N, H, W) tensor of
integer values between 0 and K-1.
"""
# Dimensions check
assert predicted.size(0) == target.size(0), \
'number of targets and predicted outputs do not match'
assert predicted.dim() == 3 or predicted.dim() == 4, \
"predictions must be of dimension (N, H, W) or (N, K, H, W)"
assert target.dim() == 3 or target.dim() == 4, \
"targets must be of dimension (N, H, W) or (N, K, H, W)"
# If the tensor is in categorical format convert it to integer format
# When the tensor is 4-dimensional, for each of the 3-dimensions of
# class for each pixel, the value it contains is confidence of that
# class. We take the "index" (class label) with the maximum confidence
# among the classes (i.e 3 dimensions)
if predicted.dim() == 4:
_, predicted = predicted.max(1)
if target.dim() == 4:
_, target = target.max(1)
# .view(-1) converts tensor to single dimensional tensor
self.conf_metric.add(predicted.view(-1), target.view(-1))
def value(self):
"""Computes the IoU and mean IoU.
The mean computation ignores NaN elements of the IoU array.
Returns:
Tuple: (IoU, mIoU). The first output is the per class IoU,
for K classes it's numpy.ndarray with K elements. The second output,
is the mean IoU.
"""
conf_matrix = self.conf_metric.value()
if self.ignore_index is not None:
for index in self.ignore_index:
conf_matrix[:, self.ignore_index] = 0
conf_matrix[self.ignore_index, :] = 0
true_positive = np.diag(conf_matrix)
false_positive = np.sum(conf_matrix, 0) - true_positive
false_negative = np.sum(conf_matrix, 1) - true_positive
# Just in case we get a division by 0, ignore/hide the error
with np.errstate(divide='ignore', invalid='ignore'):
iou = true_positive / (true_positive + false_positive +
false_negative)
return iou, np.nanmean(iou)
def main(instance_folder, results_folder, remap_file=None, parameters={'-c': 1}, multilabel=None, pattern=None, silent=False):
'''
Returns a list of lists of predicted labels (per fold).
Contains the main pipeline:
* Remaps the class labels if a mapping is specified.
* If a specific label is specified in <multilabel>, assume we're
dealing with a multilabel dataset and only consider the given label.
* For each fold, trains and tests each fold based on the instances in
<instance_folder>. By default the script looks for files with the following
filenames:
N_GRAM_TOKEN_?1.TotalSet.ngrams.txt or
Use the <pattern> option to specify your own pattern (can be a regex).
'''
assert not (multilabel and remap_file), '--multilabel and --remap cannot be used at the same time'
instance_files = getInstances(instance_folder, pattern=pattern)
labels = getLabelMap(instance_folder)
'''
Get all class (re-)mappings.
Example:
original label original SVM remapped remapped SVM
A 1 X 1
B 2 Y 2
C 3 Y 2
D 4 Y 2
The stylene_mapping maps from the original label to its SVM equivalent
The intermediate mapping maps from "orginal SVM" to "remapped SVM"
The remapping maps from the original label to the remapped labels
The true_mapping maps from "remapped" to its SVM equivalent
'''
true_mapping = {}
stylene_mapping = getLabelMap(instance_folder)
stylene_start_idx = min(stylene_mapping.values())
stylene_max_idx = max(stylene_mapping.values())
if remap_file:
intermediate_mapping = {}
remapping = getRemapping(remap_file)
assert not False in [(l in remapping) for l in stylene_mapping], "Your label remapping file doesn't contain all classes found in the Stylene mapping!"
for i, label in enumerate(sorted(set(remapping.values()))):
# Set instances to be excluded.
if label == 'EXCLUDE':
true_mapping[label] = 0
else:
if stylene_start_idx == 0:
true_mapping[label] = i+1
else:
true_mapping[label] = i
for label, mapped in stylene_mapping.items():
intermediate_mapping[mapped] = true_mapping[remapping[label]]
elif multilabel:
remapping = {}
stylene_mapping['other'] = stylene_max_idx + 1
assert multilabel in stylene_mapping, "The --multilabel label doesn't seem to be present in the Stylene mapping!"
for label in stylene_mapping:
if label == multilabel:
remapping[label] = label
else:
remapping[label] = 'other'
true_mapping[multilabel] = stylene_mapping[multilabel]
true_mapping['other'] = stylene_max_idx + 1
intermediate_mapping = {}
for label, mapped in stylene_mapping.items():
intermediate_mapping[mapped] = true_mapping[remapping[label]]
multilabel = (multilabel, stylene_mapping[multilabel])
else:
intermediate_mapping = None
true_mapping = stylene_mapping
# Invert for reverse lookup
true_mapping = invertLabelMap(true_mapping)
gold_labels = []
predicted_labels = []
predicted_values = defaultdict(list)
global_cm = ConfusionMatrix()
fold_accuracies = []
# Train and test each fold.
for i, (train_file, test_file) in enumerate(instance_files):
# If the dictionary file exists, use it to add more meta-information.
base_name = test_file.split('.ngrams.txt')[0]
if base_name == test_file:
dict_file = os.path.splitext(test_file)[0] + '.filelist.txt'
else:
dict_file = base_name + '.dictionary.ngrams.txt'
base_name = os.path.basename(base_name.split('.TotalSet')[0])
if not os.path.exists(dict_file):
original_filenames = None
else:
original_filenames = []
with open(dict_file, 'r') as fin:
while True:
line = fin.readline()
if not line.strip():
break
original_filenames.append(line.strip().split()[1])
print 'Processing fold {0}/{1}'.format(i+1, len(instance_files))
# Train!
fold_model, fold_distribution = train(train_file, i, intermediate_mapping, parameters, multilabel, results_folder)
if not silent:
print 'Training distribution:', dict([(true_mapping[label], dist) for label, dist in fold_distribution.items()])
# Classify!
pred_labels, pred_values, label_order, test_labels = test(test_file, fold_model, i, intermediate_mapping, multilabel)
# Write the predicted (non-remapped) labels to file.
pred_values_dict = dict([(label, []) for label in label_order])
output_folder = os.path.join(results_folder, 'fold-{0:02d}'.format(i+1))
if multilabel:
predictions_file = os.path.join(output_folder, os.path.basename(test_file) + '.{0}.predicted'.format(multilabel[0]))
else:
predictions_file = os.path.join(output_folder, os.path.basename(test_file) + '.predicted')
with open(predictions_file, 'w') as fout:
# Headers
fout.write('filename predicted\t')
for label in label_order:
fout.write('{0}\t'.format(label))
fout.write('{0}\n'.format(dict([(label, true_mapping[label]) for label in label_order])))
# Predictions
for j, (pred_label, pred_vals) in enumerate(zip(pred_labels, pred_values)):
for k, value in enumerate(pred_vals):
pred_values_dict[label_order[k]].append(value)
if not original_filenames:
filename = 'unknown'
else:
filename = original_filenames[j].split('.ProcessedFrequencies.xml')[0].split(base_name)[0].rstrip('.')
fout.write('{0}\t{1}\t{2}\n'.format(filename, int(pred_label), ' '.join([str(val) for val in pred_vals])))
# Remap to the true labels
pred_labels = [true_mapping[label] for label in pred_labels]
predicted_labels.extend(pred_labels)
g_labels = [true_mapping[label] for label in test_labels]
gold_labels.extend(g_labels)
local_cm = ConfusionMatrix()
for (gold, pred) in zip(g_labels, pred_labels):
local_cm.update(gold, pred)
global_cm.update(gold, pred)
# Update the predicted values
pred_values = dict([(true_mapping[label], values) for label, values in pred_values_dict.items()])
assert i==0 or sorted(pred_values) == sorted(predicted_values)
for label, values in pred_values.items():
predicted_values[label].extend(values)
if not silent:
print local_cm
print 'Fold Accuracy: {0:.3f}'.format(local_cm.accuracy)
fold_accuracies.append(local_cm.accuracy)
avg_acc = sum(fold_accuracies)/len(fold_accuracies)
stdev = (sum([(acc - avg_acc)**2 for acc in fold_accuracies])/len(fold_accuracies))**0.5
correct = sum([global_cm.TP(c) for c in global_cm.classes])
total = len(global_cm)
if not silent:
print
print 'Global confusion matrix:'
print global_cm
print
for label in global_cm.classes:
print '{0} P\t{1:.3f}'.format(label, global_cm.precision(label))
print '{0} R\t{1:.3f}'.format(label, global_cm.recall(label))
print '{0} F\t{1:.3f}'.format(label, global_cm.fmeasure(label))
print
print 'Accuracy: {0:.3f} +/- {1:.3f} ({2}/{3})'.format(global_cm.accuracy, stdev, correct, total)
print 'Micro-avg F:', global_cm.microfmeasure()
print 'Macro-avg F:', global_cm.macrofmeasure()
return global_cm
start = time.time()
y_test = []
y_pred = []
for j in range(len(data_test[i]["tweet"])):
prediction = nb.predict(data_test[i]["tweet"][j],
data_test[i]["target"][j])
y_test.append(data_test[i]["target"][j])
y_pred.append(prediction)
print("nb pred")
print(time.time() - start)
start = time.time()
cm = ConfusionMatrix()
accuracy, precision, recall, fmeasure = cm.score(y_test, y_pred)
print("Stopwords")
print(stopwords)
print("\nRemoved Stopwords")
print(removed_words)
print("\nAccuracy : {}".format(accuracy))
print("Precision : {}".format(precision))
print("Recall : {}".format(recall))
print("FMeasure : {}".format(fmeasure))
# df = pd.DataFrame({'X':x_array,'Y':y_array,'L':l_array,'K-Fold':kfold_per_combination,'Accuracy':list_acc,'Precision':list_prec,'Recall':list_recall,'F-Measure':list_fmeasure,'Fold Accuracy':fold_accuracy,'Fold Precision':fold_precision,'Fold Recall':fold_recall,'Fold F-Measure':fold_fmeasure})
# print(df)
# df.to_excel(r'cobabarunih.xlsx', index = False, header=True)
num_epochs = 150
train_acc = []
valid_acc = []
cur_loss = 0
loss = []
valid_loss = []
Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand")
Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand")
import gc
for epoch in range(num_epochs):
cur_loss = 0
val_loss = 0
confusion_valid = ConfusionMatrix(2)
confusion_train = ConfusionMatrix(2)
for im in Train:
XY, XZ, YZ, Pos, Y_train = DP.Patch_triplanar_para(im, PS)
num_samples_train = Y_train.shape[0]
num_batches_train = num_samples_train // batch_size
for i in range(num_batches_train):
idx = range(i * batch_size, (i + 1) * batch_size)
xy_batch = XY[idx]
xz_batch = XZ[idx]
yz_batch = YZ[idx]
pos_batch = Pos[idx]
target_batch = Y_train[idx]
batch_loss = f_train(xy_batch, xz_batch, yz_batch,
y_test,
writer=dev_summary_writer)
if accuracy > max_acc:
max_acc = accuracy
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
else:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
saver.restore(sess, ckpt.model_checkpoint_path)
accuracy, predictions = dev_step(x_test, y_test)
#dev_step(x_dev, y_dev, writer=dev_summary_writer)
outputs = np.argmax(predictions, axis=-1)
print(outputs.shape)
confusion_test = ConfusionMatrix(num_classes)
y_test = np.argmax(y_test, axis=-1)
print(y_test.shape)
confusion_test.batch_add(y_test, outputs)
test_accuracy = confusion_test.accuracy()
a, positive_predictive_value = confusion_test.positive_predictive_value()
b, negative_predictive_value = confusion_test.negative_predictive_value()
e, F1 = confusion_test.F1()
f, MCC = confusion_test.matthews_correlation()
cf_val = confusion_test.ret_mat()
print("FINAL TEST RESULTS")
print(confusion_test)
print(cf_val)
def run_blindset(topic, results_type, conf_matrix, blindset_name):
"""
Runs blindset test using credentials in ../Credentials.py
"""
# get credentials, import + export folders
import Credentials
active_adoption = Credentials.active_adoption
instance_creds = Credentials.ctx[active_adoption]
print(instance_creds)
print('print works')
workspace_id = Credentials.workspace_id[active_adoption][topic]
workspace_thresh = Credentials.calculate_workspace_thresh(topic)
conversation_version = Credentials.conversation_version
# import + export folders
import config
import time
data_folder = config.data_dir
export_folder = config.output_folder
timestr = time.strftime("%Y%m%d-%H%M")
blindset_name = blindset_name or topic + "_blindset.csv"
output_loc_results = os.path.join(
export_folder, "{}_results_raw_{}.csv".format(topic, timestr))
output_loc_metrics = os.path.join(
export_folder, "{}_results_metrics_{}.csv".format(topic, timestr))
output_loc_confmat = os.path.join(
export_folder, "{}_confmat_{}.png".format(topic, timestr))
# authenticate
if 'apikey' in instance_creds:
logger.debug("Authenticating (apikey)")
bs = blindset(apikey=instance_creds['apikey'],
url=instance_creds['url'],
threshold=workspace_thresh,
version=conversation_version)
elif 'password' in instance_creds:
logger.debug("Authenticating (username/password)")
bs = blindset(username=instance_creds['username'],
password=instance_creds['password'],
url=instance_creds['url'],
threshold=workspace_thresh,
version=conversation_version)
# run test
blindset_df = bs.import_blindset(os.path.join(data_folder, blindset_name))
# TODO: check blindset df
results = bs.run_blind_test(blindset_df, workspace_id)
# exports + metrics
if (results_type == 'raw') or (results_type == 'all'):
cols_export = [
col for col in results.columns.values if col != 'intent_correct'
]
results[cols_export].to_csv(output_loc_results, encoding='utf-8')
logger.info("Raw results exported to {}".format(output_loc_results))
if (results_type == 'metrics') or (results_type == 'all'):
met = Metrics(workspace_thresh)
metric_df, _ = met.get_all_metrics(results, detailed_results=True)
metric_df.to_csv(output_loc_metrics, encoding='utf-8')
logger.info(
"Metrics per intent exported to {}".format(output_loc_metrics))
# confusion matrix
if conf_matrix:
from confusionmatrix import ConfusionMatrix
cfn = ConfusionMatrix(workspace_thresh=workspace_thresh)
cfn.create(results, fig_path=output_loc_confmat)
#bs.plot_confusion_matrix(results, output_loc_confmat)
logger.info("Confusion matrix saved to {}".format(output_loc_confmat))
# print high-level metrics
overall_metrics = bs.calculate_overall_metrics(results,
av_method="weighted")
logger.info("Overall metrics for the workspace (weighted):")
logger.info(overall_metrics)
