def _parse_function_inference(self, filename, label): """Input parser for samples of the validation/test set.""" # convert label number into one-hot-encoding one_hot = tf.one_hot(label, self.num_classes) tool = victorinox() # load and preprocess the image img_string = tf.read_file(filename) #img_decoded = tf.image.decode_png(img_string) img_decoded = tf.image.decode_jpeg(img_string, channels=3) # h = img_decoded.get_shape().as_list()[0] # if h is None: # h = 227 # w = img_decoded.get_shape().as_list()[crossval1.old] # if w is None: # w = 227 # h, w = tool.get_convolvable_dim(h, w) # img_resized = tf.image.resize_images(img_decoded, [h, w]) img_resized = tf.image.resize_images(img_decoded, [RESIZE_DIM[0], RESIZE_DIM[1]]) #img_centered = tf.subtract(img_resized, IMAGENET_MEAN) #img_centered = tf.subtract(img_decoded, IMAGENET_MEAN) img_centered = tf.subtract( img_resized, self.mean_pixels) #OCRD_RELOC_UNRELOC_227_MEAN) # RGB -> BGR img_bgr = img_centered[:, :, ::-1] #img_bgr = img_decoded[:, :, ::-crossval1.old] #img_bgr = img_resized[:, :, ::-crossval1.old] return img_bgr, one_hot
nltk.download('stopwords') from nltk.stem import PorterStemmer from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory, StopWordRemover, ArrayDictionary from Sastrawi.Stemmer.StemmerFactory import StemmerFactory import os import pandas as pd import re import PyPDF2 from collections import Counter import math import random ct = CRFTagger() ct.set_model_file('all_indo_man_tag_corpus_model.crf.tagger') tool = victorinox() population1_dict = {} population2_dict = {} population_root_path = r"corpus/population" population_files = glob(os.path.join(population_root_path, "**/*.txt"), recursive=True) tokenizer = RegexpTokenizer(r'[a-zA-Z]+') factory = StemmerFactory() stemmer = factory.create_stemmer() default_stopwords = StopWordRemoverFactory().get_stop_words() additional_stopwords = [ "(", ")", "senin", "selasa", "rabu", "kamis", "jumat", "sabtu", "minggu" ] dictionary = ArrayDictionary(default_stopwords + additional_stopwords) id_stopword = StopWordRemover(dictionary) en_stopword = set(stopwords.words('english'))
def classify_by_knn_tensorflow(self, csv_train, csv_test, k, batch, result_csv): #X_t, y_t, x_t, k_t): dftrain = pd.read_csv(csv_train, sep=",", header=None) dftest = pd.read_csv(csv_test, sep=",", header=None) npdtrain = np.array(dftrain) #[:10]) npdtest = np.array(dftest) #[:5]) X_t = npdtrain[:, :-1] y_t = npdtrain[:, -1] x_t = npdtest[:, :-1] gt_label = npdtest[:, -1] k_t = k X_t_shape = np.shape(X_t) x_t_shape = np.shape(x_t) neg_one = tf.constant(-1.0, dtype=tf.float32) self.Xtrain_ph = tf.placeholder(dtype=tf.float32, shape=[X_t_shape[0], 48], name="Xtrain") #X_t_shape[1] self.Xtest_ph = tf.placeholder(dtype=tf.float32, shape=[48], name="Xtest") #x_t_shape[1] self.save_batch = tf.placeholder(dtype=tf.int32, name="save_batch") self.k_neighbor = tf.placeholder(dtype=tf.int32, name="k_neighbor") with open(result_csv, "a+") as f: pass results = [] times = [] distances = self.get_average_of_JensenShannon_using_tensorflow( batch=self.save_batch, Xtrains=self.Xtrain_ph, Xtest=self.Xtest_ph) # to find the nearest points, we find the farthest points based on negative distances # we need this trick because tensorflow has top_k api and no closest_k or reverse=True api neg_distances = tf.multiply(distances, neg_one) # get the indices vals, indx = tf.nn.top_k(neg_distances, self.k_neighbor) # slice the labels of these points y_s = tf.gather(y_t, indx) # we compute the L-1 distance #distances = tf.reduce_sum(tf.abs(tf.subtract(X_t, x_t)), 1) for i in range(len(x_t)): x_test = x_t[i] gtlabel = gt_label[i] config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=config) as sess: #x_test=tf.reshape(x_test,shape=[48]) begin = time.time() out = sess.run(y_s, feed_dict={ self.Xtrain_ph: X_t, self.Xtest_ph: x_test, self.save_batch: batch, self.k_neighbor: k_t }) end = time.time() elapsed = end - begin print "TIME=%f" % (elapsed) print "GT/predict = %s/%s %r" % (gtlabel, out, gtlabel == out) results.append([gtlabel, out[0]]) times.append(elapsed) #print self.get_label(out) # config = tf.ConfigProto(allow_soft_placement=True) # with tf.Session(config=config) as sess: # out=sess.run(y_s) # print self.get_label(out) tool = victorinox() npres = np.array(results) tool.calculate_f1_measure(npres[:, 0], npres[:, -1]) head, tail = os.path.split(result_csv) times_csv = os.path.join(head, "times.csv") if len(results) > 0: np.savetxt(result_csv, results, fmt="%s", delimiter=",") if len(times) > 0: np.savetxt(times_csv, times, fmt="%s", delimiter=",") return #y_s
times_csv = "/home/andri/Documents/s2/5/master_arbeit/dataset/uw3/unrelocated/augmented/times.csv" ocrd_aug_all = "/home/andri/Documents/s2/5/master_arbeit/dataset/ocrd/unrelocated/227/augmented.csv" ocrd_aug_train = "/home/andri/Documents/s2/5/master_arbeit/dataset/ocrd/unrelocated/227/augmented/train.csv" ocrd_aug_test = "/home/andri/Documents/s2/5/master_arbeit/dataset/ocrd/unrelocated/227/augmented/test.csv" ocrd_aug_rlbwxyh_train = "ocrd_aug_rlbwxh_train.csv" ocrd_aug_rlbwxyh_test = "ocrd_aug_rlbwxh_test.csv" ocrd_aug_rlbwxyh_classification_result_csv = "ocrd_aug_rlbwxh_classification_result.csv" ocrd_aug_rlbwxyh_classification_times_csv = "ocrd_aug_rlbwxh_classification_times.csv" # g.convert_images_to_rlbwxyh_from_file_csv(img_folder_csv=ocrd_aug_test,#ocrd_aug_train,#uw3_aug_test,#uw3_aug_train, # csv_path=ocrd_aug_rlbwxyh_test,#ocrd_aug_rlbwxyh_train,#uw3_aug_rlbwxyh_test,#uw3_aug_rlbwxyh_train,#uw3_aug_rlbwxh_result, # text_resume_idx=103980,#71750,#0,#415850,#358040,#309090,#73310,#0,#9600,#0,#428100,#413100,#337600,#92400,#0, #312100,#233100,#197300,#21300,#0,###################519800,#508500,#326900,#181700,#0, # save_batch=1)#10)#10)#00)#1)# vic = victorinox() uw3_dict = vic.convert_csv_to_dictionary(uw3_dct_csv) ocrd_dict = vic.convert_csv_to_dictionary(ocrd_dct_csv) # dff=pd.read_csv(uw3_rlbwxyh_classification_result_csv,sep=",",header=None) dff = pd.read_csv(ocrd_aug_rlbwxyh_classification_result_csv, sep=",", header=None) npd = np.array(dff) resume_idx = len(npd) # knn.classifyKNN_by_rlbwxyh_by_train_val_csv(train_csv=uw3_aug_rlbwxyh_train,#uw3_aug_rlbwxyh,# # test_csv=uw3_aug_rlbwxyh_test, # class_dict=uw3_dict, # result_csv=uw3_rlbwxyh_classification_result_csv,#aug_res, # times_csv=times_csv, # resume_test_idx=resume_idx,#4870,#4510,#0,#9650,#6970,#4360,#3360,#910,#610,#0,
def knn_test(self, train_csv="/a.csv", test_csv="/a.csv", batch_size=32, num_classes=21, shuffle=True, mean_pixels=[127, 127, 127]): tool = victorinox() # with tf.device('/gpu:0'):#with tf.device('/cpu:0'): # tr_data = ImageDataGenerator(train_csv, # mode='training', # batch_size=batch_size, # num_classes=num_classes, # shuffle=shuffle, # mean_pixels=mean_pixels) # # val_data = ImageDataGenerator(test_csv, # mode='inference', # batch_size=batch_size, # num_classes=num_classes, # shuffle=False, # mean_pixels=mean_pixels) # # create an reinitializable iterator given the dataset structure # iterator = Iterator.from_structure(tr_data.data.output_types, # tr_data.data.output_shapes) # next_batch = iterator.get_next() # training_init_op = iterator.make_initializer(tr_data.data) # validation_init_op = iterator.make_initializer(val_data.data) # Import MNIST data # from tensorflow.examples.tutorials.mnist import input_data # mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) # In this example, we limit mnist data Xtr, Ytr = self.get_data( train_csv, sep="," ) #mnist.train.next_batch(5000) # 5000 for training (nn candidates) Xte, Yte = self.get_data( test_csv, sep=",") #mnist.test.next_batch(200) # 200 for testing # tf Graph Input xtr = tf.placeholder(tf.float32, [None, np.shape(Xtr)[1]]) xte = tf.placeholder(tf.float32, np.shape(Xte)[1]) #[784]) from tensorflow.contrib.bayesflow.python.ops.csiszar_divergence import jensen_shannon # Nearest Neighbor calculation using L1 Distance # Calculate L1 Distance #distance = tf.reduce_sum(tf.abs(tf.add(xtr, tf.negative(xte))), reduction_indices=1) distance = tool.get_average_of_JensenShannon_using_tensorflow(xtr, xte) # Prediction: Get min distance index (Nearest neighbor) pred = tf.arg_min(distance, 0) accuracy = 0. # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() # Start training with tf.Session() as sess: # Run the initializer sess.run(init) # loop over test data for i in range(len(Xte)): # Get nearest neighbor nn_index = sess.run(pred, feed_dict={xtr: Xtr, xte: Xte[i, :]}) # Get nearest neighbor class label and compare it to its true label print("Test", i, "Prediction:", np.argmax(Ytr[nn_index]), \ "True Class:", np.argmax(Yte[i])) # Calculate accuracy if np.argmax(Ytr[nn_index]) == np.argmax(Yte[i]): accuracy += 1. / len(Xte) print("Done!") print("Accuracy:", accuracy)