Ejemplo n.º 1
0
class NERValidator(object):
    def __init__(self, recbysns, classifiers):
        self.recbysns = recbysns
        self.db = self.recbysns.db
        self.classifiers = classifiers
        self.evaluator = Evaluator()
        self.confusion_matrixes = {str(classifier):
                                   [] for classifier in self.classifiers}

    def validate(self):
        entities = [NEREntity(self.recbysns, entity)
                    for entity in self.db.select_table('recbysns_entity', '1')]
        for classifier in self.classifiers:
            self.test(classifier, entities)
        self.evaluator.evaluate(self.confusion_matrixes)

    def test(self, classifier, entities):
        confusion_matrix = {
                NER_BOOK:   {NER_BOOK: float(0), NER_MOVIE: float(0),
                             NER_VIDEO: float(0), NER_OTHERS: float(0)},
                NER_MOVIE:  {NER_BOOK: float(0), NER_MOVIE: float(0),
                             NER_VIDEO: float(0), NER_OTHERS: float(0)},
                NER_VIDEO:  {NER_BOOK: float(0), NER_MOVIE: float(0),
                             NER_VIDEO: float(0), NER_OTHERS: float(0)},
                NER_OTHERS: {NER_BOOK: float(0), NER_MOVIE: float(0),
                             NER_VIDEO: float(0), NER_OTHERS: float(0)},
        }
        for entity in entities:
            # predicted ner class
            p_ner_class = classifier.predict(entity)
            ner_class = entity.ner_class()
            confusion_matrix[ner_class][p_ner_class] += 1
        print confusion_matrix
        self.confusion_matrixes[str(classifier)].append(confusion_matrix)
Ejemplo n.º 2
0
	def detect(self, img_path=None, output_file_prefix='', num_ft=100, offset=0, scaling_factor = 1.2, scaling_iters=3, nms=0.5, clf=None, templates=None, linear_scaling=False):
		
		#=====[ Load our classifier and templates ]=====
		clf = pickle.load(open(clf)) if clf else pickle.load(open('classifiers/top_ft_classifier_100_200', 'r'))
		templates = pickle.load(open(templates)) if templates else pickle.load(open('processed_data/top_templates_1000.p','r'))
	
		#=====[ Get top templates ]=====
		templates = templates[:num_ft]

		#=====[ Instantiate our feature generator ]=====
		fg = FeatureGenerator(templates)

		#=====[ Instantiate our detector ]=====
		if linear_scaling:
			self.detector = LinearScaleDetector(clf.clf, fg,scaling_factor=scaling_factor,scaling_iters=scaling_iters, nms=nms)
		else:
			self.detector = Detector(clf.clf, fg,scaling_factor=scaling_factor,scaling_iters=scaling_iters, nms=nms)

		#=====[ If a specific image path is given, then we do not evaluate, just detect the pedestrian and draw bounding boxes ]=====
		if img_path:
			_, bbs = self.detector.detect_pedestrians(img_path)
			self._draw_bbs(img_path, bbs)
		
		else:
			#=====[ Instantiate our evaluator and evaluate ]=====
			evaluator = Evaluator('INRIAPerson/Test', self.detector)
			FPPI, miss_rate = evaluator.evaluate(output_file_prefix,offset)

			print '\nFPPI: {}\nMiss rate: {}\n'.format(FPPI, miss_rate)
Ejemplo n.º 3
0
Archivo: sa.py Proyecto: jjyao/recbysns
class SAValidator(object):
    def __init__(self, recbysns, classifiers):
        self.recbysns = recbysns
        self.db = self.recbysns.db
        self.classifiers = classifiers
        self.evaluator = Evaluator()
        self.confusion_matrixes = {str(classifier): []
                                   for classifier in self.classifiers}

    def validate(self):
        entities = [SAEntity(self.recbysns, entity)
                    for entity in self.db.select_table('recbysns_entity',
                    "type in (%d,%d,%d)" % (NER_BOOK, NER_MOVIE, NER_VIDEO))]
        for classifier in self.classifiers:
            self.test(classifier, entities)
        self.evaluator.evaluate(self.confusion_matrixes)

    def test(self, classifier, entities):
        confusion_matrix = {
                SA_POSITIVE: {SA_POSITIVE: float(0), SA_NETURAL: float(0),
                              SA_NEGATIVE: float(0)},
                SA_NETURAL:  {SA_POSITIVE: float(0), SA_NETURAL: float(0),
                              SA_NEGATIVE: float(0)},
                SA_NEGATIVE: {SA_POSITIVE: float(0), SA_NETURAL: float(0),
                              SA_NEGATIVE: float(0)},
        }
        for entity in entities:
            # predicted sa_class
            p_sa_class = classifier.predict(entity)
            # actual sa_class
            sa_class = entity.sa_class()
            confusion_matrix[sa_class][p_sa_class] += 1
        print confusion_matrix
        self.confusion_matrixes[str(classifier)].append(confusion_matrix)
Ejemplo n.º 4
0
Archivo: parser.py Proyecto: k0ner/sil
 def p_interpreter_other(self, p):
     '''interpreter : comparision
                    | select
                    | array_selection
                    | func_call_stmt
                    | expression'''
     print Evaluator.visit(p[1])
Ejemplo n.º 5
0
def __main__(argv):
    #%%
    logger = logging.getLogger(__name__)
    logger.info("VECTOR MODEL INFORMATION RETRIEVAL SYSTEM START")    
    
    gli = InvertedIndexGenerator(GLI_CONFIG_FILE)
    gli.run()
    gli.write_output()
    
    index = Indexer(INDEX_CONFIG_FILE, TfidfVectorizer)
    index.run()
    index.write_output()
    
    pc = QueryProcessor(PC_CONFIG_FILE)
    pc.run()
    pc.write_output()
    
    buscador = SearchEngine(BUSCA_CONFIG_FILE, TfidfVectorizer)
    buscador.run()
    buscador.write_output()
    #%%
    avaliador = Evaluator(AVAL_CONFIG_FILE)
    avaliador.run()
    avaliador.write_output()
    
    logger.info("VECTOR MODEL INFORMATION RETRIEVAL SYSTEM DONE")     
def evaluate_classifier(clf_wrapper,
                        params,
                        train_images_codelabels, train_labels,
                        test_images_codelabels, test_labels):

    print "==\nevaluate_classifier (test size: {})\n{}".format(len(test_labels), clf_wrapper)

    print "training classifier {}".format(clf_wrapper.clf)
    start_time = time.time()
    clf_wrapper.fit(X=train_images_codelabels, labels=train_labels)
    et = (time.time() - start_time) * 1000.0
    print "finished training classifier - took {}ms".format(et)

    # evaluate
    print "proceeding to evaluate classifier on test set {}".format(len(test_labels))
    encoded_test_labels = clf_wrapper.label_encoder.transform(test_labels)

    evaluator = Evaluator(
        clf=clf_wrapper.clf,
        label_encoder=clf_wrapper.label_encoder,
        params=params,
        output_filepath="../results/evaluation_results_{}.json".format(clf_wrapper)
    )

    evaluator.results["classifier"] = "{}".format(clf_wrapper.clf)
    evaluator.results["classifier_training_time"] = "{}".format(et)

    evaluation_results = evaluator.evaluate(X=test_images_codelabels, y=encoded_test_labels)
    print evaluation_results
Ejemplo n.º 7
0
 def run(self):
     while 1:
         try:
             s = raw_input('calc> ')
         except EOFError:
             break
         if not s: continue
         Evaluator.visit(yacc.parse(s))
def window_overlap_test(window_overlap=2.):
    """  """

    train_labels, train_images, test_labels, test_images = get_training_and_test_data()

    # split to make experimentation quicker
    train_labels, train_images = get_subset_of_training_data(train_labels, train_images, split=0.5)

    training_size = len(train_labels)

    desc = "testing influence of window_overlap, set to {}. NB training size = {}".format(
        window_overlap,
        training_size
    )

    print desc

    selected_labels = list(set(train_labels))

    params = build_params(num_classes=len(selected_labels),
                          training_size=len(train_images),
                          test_size=len(test_images),
                          window_overlap=window_overlap,
                          fn_prefix="winoverlap-{}".format(window_overlap))

    trainer = SketchRecognitionTrainer(
        file_path=SketchRecognitionTrainer.get_cookbook_filename_for_params(params=params),
        run_parallel_processors=True,
        params=params
    )

    classifier = trainer.train_and_build_classifier(train_labels, train_images)
    encoded_test_labels = classifier.le.transform(test_labels)

    test_images_codelabels = trainer.code_labels_for_image_descriptors(
        trainer.extract_image_descriptors(test_images)
    )

    evaluator = Evaluator(
        clf=classifier.clf,
        label_encoder=classifier.le,
        params=params,
        output_filepath=SketchRecognitionTrainer.get_evaluation_filename_for_params(params=params)
    )

    # add timings to output
    evaluator.results["timings"] = {}
    for key, value in trainer.timings.iteritems():
        evaluator.results["timings"][key] = value

    # add comment
    evaluator.results["desc"] = desc

    evaluation_results = evaluator.evaluate(X=test_images_codelabels, y=encoded_test_labels)
    print evaluation_results
class Test(unittest.TestCase):


    def setUp(self):
        gold_directory = os.path.join(DATA_DIRECTORY, 'segment_data_test')
        result_directory = os.path.join(DATA_DIRECTORY, 'data_test_result')
        self.evaluator = Evaluator(gold_directory, result_directory)


    def test_calculate_precision(self):
        self.evaluator.calculate_precision()
Ejemplo n.º 10
0
def main():
	if len(sys.argv) != 6:
		assert False, "INSUFFICIENT ARGUMENTS!"

	filenames = [line.strip() for line in open(sys.argv[4])]
	ann_filepaths = [sys.argv[5] + '/' + filename[:len(filename)-4]+".ann" for filename in filenames]
	# print(ann_filepaths)
	
	data_param = {"dataset_name": sys.argv[1], "ann_filepaths": ann_filepaths,"n_label": int(sys.argv[2]), "gt_img_dir": sys.argv[3], "test_img_list_filepath": sys.argv[4], "result_dir": sys.argv[5], "mapping_label": color_class_map, "dir_output": sys.argv[5]}

	eval = Evaluator(data_param)
	eval.evaluate_all()
Ejemplo n.º 11
0
def main():
    mode = argv[1]
    e = Evaluator()
    if mode == 'wikt':
        e.read_all_wiktionary()
        e.compare_with_triangles_stdin()
    elif mode == 'feat':
        e.write_labels(argv[2])
        e.featurize_and_uniq_triangles_stdin()
Ejemplo n.º 12
0
 def _process(self):
     self.msg = self._receive(True)
     if self.msg:
         request = json.loads(self.msg.content)
         if request['request_type'] == 'games':
             self.games = request['data']
             self.show_dialog()
         if request['request_type'] == 'game_evaluation':
             PrintFormatter.results(request['data'])
             Evaluator.make_bet(self.mood, request['data'])
             print "\n********Results********\n"
             Evaluator.find_result(request['data'])
             self.show_dialog()
Ejemplo n.º 13
0
def main():
    """Requests infix expressions, translates them to postfix,
    and evaluates them, until the user enters nothing."""
    while True:
        sourceStr = input("Enter an infix expression: ")
        if sourceStr == "": break
        try:
            scanner = Scanner(sourceStr)
            translator = Translator(scanner)
            postfix = translator.translate()
            evaluator = Evaluator(postfix)
            print("Value:", evaluator.evaluate())
        except Exception as e:
            print("Error:", e, translator.translationStatus())
Ejemplo n.º 14
0
    def test_end_to_end(self):
        with open("unit_tests/fixtures/tests_games.json") as file:
            data = json.loads(file.read())

        # Update player's source placeholder with actual code
        with open("unit_tests/fixtures/Snakes/Snakes.cpp") as source:
            data["source"] = source.read()

        # Update opponent's source placeholder with actual code
        with open("unit_tests/fixtures/Snakes/Opponent.cpp") as source:
            data["matches"][0]["source"] = source.read()

        evaluator = Evaluator(data)
        evaluator.evaluate()
Ejemplo n.º 15
0
    def solve(self, instance, startpoint=None):
        if startpoint is None:
            startpoint = Random().solve(instance)

        e = Evaluator(instance)
        current = (startpoint, e.evaluate(startpoint))

        while True:
            next_step = self.select_step(e, current)
            if not next_step:
                break
            else:
                current = next_step

        return current[0]
Ejemplo n.º 16
0
 def __init__(self, popSize=10, populate=False):
     self.__size = popSize
     self.__members = [None] * self.__size
     self.__eval = Evaluator()
     self.__BITS = self.__eval.getBits()
     if (populate):
         self.__populate(1, self.__BITS)
Ejemplo n.º 17
0
 def __init__(self, recbysns, classifiers):
     self.recbysns = recbysns
     self.db = self.recbysns.db
     self.classifiers = classifiers
     self.evaluator = Evaluator()
     self.confusion_matrixes = {str(classifier):
                                [] for classifier in self.classifiers}
Ejemplo n.º 18
0
	def get_stats(self, output_file_prefix='', num_images=100):
		""" Reports number of people, matches, misses, and false positives per experiment from generated output files """

		#=====[ Instantiate our evaluator ]=====
		evaluator = Evaluator('INRIAPerson/Test')

		num_people, num_hits, num_misses, num_FP, num_processed = evaluator.aggregate_stats(output_file_prefix, num_images)

		#=====[ Print statistics ]=====
		print '-----> Stats for ' + output_file_prefix + ' ( ' + str(num_processed) + '/' + str(num_images) +' processed) :\n\n'
		print 'Miss Rate: ' + str(float(num_misses)/num_people)
		print 'False Positives: ' + str(num_FP)
		print 'FPPI: ' + str(float(num_FP)/num_images)
		print 'Hits: ' + str(num_hits) 
		print 'Misses: ' + str(num_misses) 
		print 'Total People: ' + str(num_people) 
Ejemplo n.º 19
0
Archivo: sgd.py Proyecto: srush/tf-fork
 def __init__(self, decoder, train, dev, output, iter = 1):
     trainer.BaseTrainer.__init__(self, decoder, [train], output=output)
     self.iter = iter
     self.eval = Evaluator(decoder, [dev])
     self.u = 0
     #self.N = 10
     self.weight_norm = 0.0
def sanity_check():
    """ baseline test, all all parameters from experimentation """

    train_labels, train_images, test_labels, test_images = get_training_and_test_data()

    train_labels, train_images = get_subset_of_training_data(train_labels, train_images, split=0.05)

    selected_labels = list(set(train_labels))

    params = build_params(
        num_classes=len(selected_labels),
        training_size=len(train_images),
        test_size=len(test_images),
        fn_prefix="sanitycheck"
    )

    trainer = SketchRecognitionTrainer(
        file_path=SketchRecognitionTrainer.get_cookbook_filename_for_params(params=params),
        run_parallel_processors=True,
        params=params
    )

    classifier = trainer.train_and_build_classifier(train_labels, train_images)
    encoded_test_labels = classifier.le.transform(test_labels)

    test_images_codelabels = trainer.code_labels_for_image_descriptors(
        trainer.extract_image_descriptors(test_images)
    )

    evaluator = Evaluator(
        clf=classifier.clf,
        label_encoder=classifier.le,
        params=params,
        output_filepath=SketchRecognitionTrainer.get_evaluation_filename_for_params(params=params)
    )

    # add timings to output
    evaluator.results["timings"] = {}
    for key, value in trainer.timings.iteritems():
        evaluator.results["timings"][key] = value

    # add comment
    evaluator.results["desc"] = "After many iterations, this is a baseline for which tuning will benchmark from"

    evaluation_results = evaluator.evaluate(X=test_images_codelabels, y=encoded_test_labels)
    print evaluation_results
def training_size_test(split=0.5):
    """
    see what effect the training size has on the performance, initially take off 50%
    """
    print "test_2"

    train_labels, train_images, test_labels, test_images = get_training_and_test_data()

    train_labels, train_images = get_subset_of_training_data(train_labels, train_images, split=split)

    selected_labels = list(set(train_labels))

    params = build_params(num_classes=len(selected_labels),
                          training_size=len(train_images),
                          test_size=len(test_images))

    trainer = SketchRecognitionTrainer(
        file_path=SketchRecognitionTrainer.get_cookbook_filename_for_params(params=params),
        run_parallel_processors=True,
        params=params
    )

    classifier = trainer.train_and_build_classifier(train_labels, train_images)
    encoded_test_labels = classifier.le.transform(test_labels)

    test_images_codelabels = trainer.code_labels_for_image_descriptors(
        trainer.extract_image_descriptors(test_images)
    )

    evaluator = Evaluator(
        clf=classifier.clf,
        label_encoder=classifier.le,
        params=params,
        output_filepath=SketchRecognitionTrainer.get_evaluation_filename_for_params(params=params)
    )

    # add timings to output
    evaluator.results["timings"] = {}
    for key, value in trainer.timings.iteritems():
        evaluator.results["timings"][key] = value

    # add comment
    evaluator.results["desc"] = "After many iterations, this is a baseline for which tuning will benchmark from"

    evaluation_results = evaluator.evaluate(X=test_images_codelabels, y=encoded_test_labels)
    print evaluation_results
Ejemplo n.º 22
0
 def _init_eval(self):
     logging.info("Init eval")
     x_pre, x, y = [self.g0_inputs[k] for k in ['x_pre', 'x', 'y']]
     self.model.set_device('/gpu:0')
     self.evaluate = Evaluator(self.sess, self.model, self.batch_size,
                               x_pre, x, y,
                               self.data,
                               self.writer,
                               self.hparams)
Ejemplo n.º 23
0
def evaluate(classifier, testset=None, holdout=None, folds=10):
    """Create evaulator object.

    Args:
        classifier (Classifier): desired classifier to run
        testset (DataSet): testset to run classification accuracies/tests
        outfile (str): filepath of target output file
        holdout (float): desired split for the hold-out method
        folds (int): number of folds for cross validation
    """
    evaluator = Evaluator(classifier)
    if testset:
        pass
    elif holdout:
        evaluator.holdout(holdout)
    else:  # runing folds
        evaluator.cross_validate(folds)
    return evaluator
Ejemplo n.º 24
0
 def test(self, label=None):
     # Training error
     print "Training error:"
     evaluator = Evaluator(self.X_train, self.Y_train, self.W)
     #evaluator.MSE()
     evaluator.accuracy()
     #evaluator.confusion()
     # Testing error
     print "Testing error:"
     evaluator = Evaluator(self.X_test, self.Y_test, self.W)
     #evaluator.MSE()
     evaluator.accuracy()
     evaluator.confusion()
     FPR, TPR = evaluator.roc()
     plt.plot(FPR, TPR, label=label)
     plt.axis([0.0,0.5,0.5,1.0])
Ejemplo n.º 25
0
class Perceptron(BasePerceptron):

    def __init__(self, decoder, train, dev, output, iter = 1 , avg = True):
        BasePerceptron.__init__(self, decoder, [train], output=output)
        self.iter = iter
        self.avg = avg
        self.eval = Evaluator(decoder, [dev])


    @staticmethod
    def cmdline_perc(decoder):
        return Perceptron(decoder, train = FLAGS.train, dev = FLAGS.dev,
                          output = FLAGS.out, iter = FLAGS.iter, avg = FLAGS.avg )
        
    def avg_weights(self):
        return self.weights - self.allweights * (1/self.c)
        
    def train(self):

        starttime = time.time()

        print >> logs, "starting perceptron at", time.ctime()

        best_prec = 0
        for it in xrange(1, self.iter+1):

            print >> logs, "iteration %d starts..............%s" % (it, time.ctime())

            iterstarttime = time.time()
            num_updates, early_updates = self.one_pass_on_train()

            print >> logs, "iteration %d training finished at %s. now evaluating on dev..." % (it, time.ctime())
            avgweights = self.avg_weights() if self.avg else self.weights
            if FLAGS.debuglevel >= 2:
                print >> logs, "avg w=", avgweights
            self.decoder.set_model_weights(avgweights)
            prec = self.eval.eval().compute_score()
            
            print >> logs, "at iteration {0}, updates= {1} (early {4}), dev= {2}, |w|= {3}, time= {5:.3f}h acctime= {6:.3f}h"\
                  .format(it, num_updates, prec, len(avgweights), early_updates, \
                          (time.time() - iterstarttime)/3600, (time.time() - starttime)/3600.)
            logs.flush()

            if prec > best_prec:
                best_prec = prec
                best_it = it
                best_wlen = len(avgweights)
                print >> logs, "new high at iteration {0}: {1}. Dumping Weights...".format(it, prec)
                self.dump(avgweights)

            self.decoder.set_model_weights(self.weights) # restore non-avg

        print >> logs, "peaked at iteration {0}: {1}, |bestw|= {2}.".format(best_it, best_prec, best_wlen)
        print >> logs, "perceptron training of %d iterations finished on %s (took %.2f hours)"  % \
              (it, time.ctime(), (time.time() - starttime)/3600.)
def cluster_size_test(num_clusters=200):
    """  """

    train_labels, train_images, test_labels, test_images = get_training_and_test_data()

    selected_labels = list(set(train_labels))

    params = build_params(num_classes=len(selected_labels),
                          training_size=len(train_images),
                          test_size=len(test_images),
                          num_clusters=num_clusters)

    trainer = SketchRecognitionTrainer(
        file_path=SketchRecognitionTrainer.get_cookbook_filename_for_params(params=params),
        run_parallel_processors=True,
        params=params
    )

    classifier = trainer.train_and_build_classifier(train_labels, train_images)
    encoded_test_labels = classifier.le.transform(test_labels)

    test_images_codelabels = trainer.code_labels_for_image_descriptors(
        trainer.extract_image_descriptors(test_images)
    )

    evaluator = Evaluator(
        clf=classifier.clf,
        label_encoder=classifier.le,
        params=params,
        output_filepath=SketchRecognitionTrainer.get_evaluation_filename_for_params(params=params)
    )

    # add timings to output
    evaluator.results["timings"] = {}
    for key, value in trainer.timings.iteritems():
        evaluator.results["timings"][key] = value

    # add comment
    evaluator.results["desc"] = "testing influence of num_clusters, set to {}".format(num_clusters)

    evaluation_results = evaluator.evaluate(X=test_images_codelabels, y=encoded_test_labels)
    print evaluation_results
Ejemplo n.º 27
0
def start():
    movies = load_dataset()
    model = MatrixPreferenceDataModel(movies['data'])
    option = int(input("Enter: \n 1 for User Based Recommender \n 2 for Item Based Recommender \n"))
    if option != 1 and option != 2:
        print("Invalid Input")
        return
    if option == 1:
        similarity = UserSimilarity(model, cosine_distances)
        neighborhood = NearestNeighborsStrategy()
        recsys = UserBasedRecommender(model, similarity, neighborhood)

    if option == 2:
        similarity = ItemSimilarity(model, cosine_distances)
        neighborhood = ItemsNeighborhoodStrategy()
        recsys = ItemBasedRecommender(model, similarity, neighborhood)

    evaluator = Evaluator()
    all_scores = evaluator.evaluate(recsys, permutation=False)
    print all_scores
Ejemplo n.º 28
0
def _test():
    dset = create_dataset('tests/lenses.mff')
    dset.train.normalize_attributes()
    for e in dset.train.examples:
        print(e)
    classifier = NeuralNetwork(trainset=dset.train, max_error=.2, debug=True)
    evaluator = Evaluator(classifier)
    evaluator.holdout(.5)

    dset = create_dataset('tests/lenses.mff')
    dset.train.nominal_to_linear()
    print(dset)
    classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1, j=10)
    evaluator = Evaluator(classifier)
    evaluator.holdout(.2)
    dset = create_dataset('tests/test_data/iris-binary.mff')
    classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1)
    classifier.train(dset.train)
    dset = create_dataset('tests/test_data/votes.mff')
    classifier = NeuralNetwork(trainset=dset.train, debug=True, max_error=.1)
    classifier.train(dset.train)
    dset = create_dataset('tests/test_data/mushroom.mff')
    classifier = NeuralNetwork(trainset=dset.train, debug=True)
    classifier.train(dset.train)
    dset = create_dataset('tests/test_data/soybean.mff')
    classifier = NeuralNetwork(trainset=dset.train, debug=True)
    classifier.train()
Ejemplo n.º 29
0
 def test(self):
     # Training error
     print "Training error:"
     evaluator = Evaluator(self.X_train, self.Y_train, self.W)
     evaluator.MSE()
     evaluator.accuracy()
     # Testing error
     print "Testing error:"
     evaluator = Evaluator(self.X_test, self.Y_test, self.W)
     evaluator.MSE()
     evaluator.accuracy()
Ejemplo n.º 30
0
    def __init__(self):
        self.players_size = 0
        self.players_list = []
        self.dealer = 0
        self.last_raise = 0
        self.current_player = 0

        self.pot = 0

        self.current_cycle = 0
        self.end_of_first = 0  # To prevent skipping big blind player

        self.min_bet = 0
        self.small_blind_points = 5
        self.big_blind_points = 10
        self.initial_points = 200

        self.deck = None
        self.common_cards = []
        self.evaluator = Evaluator()

        self.done = False  # To be removed after command line testing
Ejemplo n.º 31
0
class MultiGPUTrainer(object):
    def __init__(self, config, vocab, algo):
        self.config = config
        self.vocab = vocab
        self.models = []
        self.config = config
        self.optim_type = config.optim
        self.learning_rate = config.learning_rate
        self.logger = logging.getLogger('qarc')

        # with tf.variable_scope("optimizer") as scope:
        self.opt = get_opt(self.optim_type, self.learning_rate)
        with tf.variable_scope(tf.get_variable_scope()) as scope:
            losses = []
            grads_list = []
            for gpu_idx in range(config.num_gpu):
                with tf.name_scope("grads_{}".format(gpu_idx)), tf.device(
                        "/gpu:{}".format(gpu_idx)):
                    model = choose_algo(algo, config, vocab)
                    self.models.append(model)
                    loss = model.get_loss()
                    grads = self.opt.compute_gradients(loss)
                    losses.append(loss)
                    grads_list.append(grads)
                    tf.get_variable_scope().reuse_variables()
        self.global_step = self.models[0].global_step
        self.summary_op = self.models[0].summary_op
        self.loss = tf.add_n(losses) / len(losses)
        self.grads = average_gradients(grads_list)
        self.train_op = self.opt.apply_gradients(self.grads,
                                                 global_step=self.global_step)

        sess_config = tf.ConfigProto()
        sess_config.gpu_options.allow_growth = True
        sess_config.allow_soft_placement = True
        self.sess = tf.Session(config=sess_config)
        self.sess.run(tf.global_variables_initializer())
        self.saver = tf.train.Saver()
        self.train_writer = tf.summary.FileWriter(
            os.path.join(config.summary_dir, 'train'), self.sess.graph)
        self.dev_writer = tf.summary.FileWriter(
            os.path.join(config.summary_dir, 'dev'), self.sess.graph)
        self.evaluator = Evaluator(self.config, self.models[0], self.sess,
                                   self.saver)
        self.local_global_step = 1

    def get_train_op(self):
        return self.train_op

    def step(self, batches, get_summary=True):
        feed_dict = {}
        for batch, model in zip(batches, self.models):
            feed_dict.update(get_feed_dict(model, batch))
        # print(feed_dict)
        if get_summary:
            loss, summary, train_op = \
                self.sess.run([self.loss, self.summary_op, self.train_op],
                              feed_dict=feed_dict)
            # print(start_loss.shape)
        else:
            loss, train_op = self.sess.run([self.loss, self.train_op],
                                           feed_dict=feed_dict)
            summary = None
        return loss, summary, train_op

    def _train_epoch(self, train_batches):
        """
        Trains the model for a single epoch.
        Args:
            train_batches: iterable batch data for training
            dropout_keep_prob: float value indicating dropout keep probability
        """
        total_num, total_loss = 0, 0
        log_every_n_batch, n_batch_loss = 50, 0
        multi_batches = []
        for bitx, batch in enumerate(train_batches, 1):
            multi_batches.append(batch)
            if bitx % len(self.models) != 0:
                continue
            loss, summary, train_op = self.step(multi_batches)
            self.train_writer.add_summary(summary, self.local_global_step)
            self.local_global_step += 1
            total_loss += loss * len(batch['raw_data'])
            total_num += len(batch['raw_data'])
            n_batch_loss += loss
            if log_every_n_batch > 0 and bitx % log_every_n_batch == 0:
                self.logger.info(
                    'Average loss from batch {} to {} is {}'.format(
                        bitx - log_every_n_batch + 1, bitx,
                        n_batch_loss / log_every_n_batch))
                n_batch_loss = 0
            multi_batches = []
        return 1.0 * total_loss / total_num

    def train(self,
              data,
              epochs,
              batch_size,
              save_dir,
              save_prefix,
              evaluate=True):
        """
        Train the model with data
        Args:
            data: the BRCDataset class implemented in dataset.py
            epochs: number of training epochs
            batch_size:
            save_dir: the directory to save the model
            save_prefix: the prefix indicating the model type
            dropout_keep_prob: float value indicating dropout keep probability
            evaluate: whether to evaluate the model on test set after each epoch
        """
        pad_id = self.vocab.get_word_id(self.vocab.pad_token)
        max_avg = 0
        # global_step = 0
        for epoch in range(1, epochs + 1):
            self.logger.info('Training the model for epoch {}'.format(epoch))
            train_batches = data.gen_mini_batches('train',
                                                  batch_size,
                                                  pad_id,
                                                  shuffle=True)
            train_loss = self._train_epoch(train_batches)
            self.logger.info('Average train loss for epoch {} is {}'.format(
                epoch, train_loss))

            if evaluate:
                self.logger.info(
                    'Evaluating the model after epoch {}'.format(epoch))
                if data.dev_set is not None:
                    eval_batches = data.gen_mini_batches('dev',
                                                         batch_size,
                                                         pad_id,
                                                         shuffle=False)
                    eval_loss, f1, em, avg = self.evaluator.evaluate(
                        eval_batches,
                        result_dir=self.config.result_dir,
                        result_prefix='dev.predicted')
                    self.logger.info('Dev eval loss {}'.format(eval_loss))
                    self.logger.info(
                        'Dev eval result: F1: {:.3f} EM: {:.3f} AVG: {:.3f}'.
                        format(f1, em, avg))

                    if avg > max_avg:
                        self.save(save_dir, save_prefix)
                        max_avg = avg
                else:
                    self.logger.warning(
                        'No dev set is loaded for evaluation in the dataset!')
            else:
                self.save(save_dir, save_prefix + '_' + str(epoch))

    def save(self, model_dir, model_prefix):
        """
        Saves the model into model_dir with model_prefix as the model indicator
        """
        self.saver.save(self.sess, os.path.join(model_dir, model_prefix))
        self.logger.info('Model saved in {}, with prefix {}.'.format(
            model_dir, model_prefix))

    def restore(self, model_dir, model_prefix):
        """
        Restores the model into model_dir from model_prefix as the model indicator
        """
        self.saver.restore(self.sess, os.path.join(model_dir, model_prefix))
        self.logger.info('Model restored from {}, with prefix {}'.format(
            model_dir, model_prefix))
Ejemplo n.º 32
0
    for definition in definitions:
        definition = definition
        words = definition.split(' ')
        def_embedding = np.zeros_like(embeddings_dict['a'])

        for word in words:
            if word not in stop_words and word not in punctuation and word in embeddings_dict:
                def_embedding += embeddings_dict[word]

        def_embeddings.append(def_embedding)

    return def_embeddings


eval = Evaluator()
glove_dict = eval.load_glove_embeddings()

#TODO: we can use two dict of embeddings, one for definition (all words), one just for lexical items in the dictionary
validate_dict = {}
with open('../data/data_train_words.txt') as f:
    lines = f.readlines()
lines = [line[:-1] for line in lines]
words = sorted(list(set(lines)))
for word in words:
    if word in glove_dict:
        validate_dict[word] = glove_dict[word]

def_embeddings = create_definition_embedding(
    glove_dict, "../data/data_train_definitions.txt")
print("loaded_embedding")
Ejemplo n.º 33
0
    # input random seed
    if args.seed > 0:
        np.random.seed(args.seed)
        env.seed(args.seed)

    # input states count & actions count
    print(env.observation_space.shape, env.action_space.shape)
    nb_states = env.observation_space.shape[0]
    if args.discrete:
        nb_actions = env.action_space.n
    else:
        nb_actions = env.action_space.shape[0]

    env = fastenv(env, args.action_repeat, args.vis)
    agent = DDPG(nb_states, nb_actions, args, args.discrete, args.cuda)
    evaluate = Evaluator(args.validate_episodes,
                         max_episode_length=args.max_episode_length)

    if args.vis and args.env == 'HalfCheetahBulletEnv-v0':
        env.render()

    if args.test is False:
        train(args.train_iter,
              agent,
              env,
              evaluate,
              args.validate_interval,
              args.output,
              args.window_length,
              max_episode_length=args.max_episode_length,
              debug=args.debug,
              visualize=args.vis,
Ejemplo n.º 34
0
model = build_model(config, source_dictionary.vocabulary_size, target_dictionary.vocabulary_size)

predictor = Predictor(
    preprocess=IndexedInputTargetTranslationDataset.preprocess(source_dictionary),
    postprocess=lambda x: ' '.join([token for token in target_dictionary.tokenify_indexes(x) if token != '<EndSent>']),
    model=model,
    checkpoint_filepath=args.checkpoint
)

timestamp = datetime.now()
if args.save_result is None:
    eval_filepath = 'logs/eval-{config}-time={timestamp}.csv'.format(
        config=args.config.replace('/', '-'),
        timestamp=timestamp.strftime("%Y_%m_%d_%H_%M_%S"))
else:
    eval_filepath = args.save_result

evaluator = Evaluator(
    predictor=predictor,
    save_filepath=eval_filepath
)

print('Evaluating...')
test_dataset = TranslationDataset(config['data_dir'], args.phase, limit=1000)
bleu_score = evaluator.evaluate_dataset(test_dataset)
print('Evaluation time :', datetime.now() - timestamp)

print("BLEU score :", bleu_score)


Ejemplo n.º 35
0
class Trainer(object):
    def __init__(self, args):
        np.random.seed(args.seed)

        self.args = args

        self.logger = logger.Logger(args.output_dir)
        self.args.logger = self.logger

        current_commit_hash =\
            subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
        self.logger.log('current git commit hash: %s' % current_commit_hash)

        print('load vec')
        source_vecs, source_dico =\
            utils.load_word_vec_list(args.source_vec_file, args.source_lang)
        target_vecs, target_dico =\
            utils.load_word_vec_list(args.target_vec_file, args.target_lang)

        self.src_dico = source_dico
        self.tgt_dico = target_dico
        args.src_dico = source_dico
        args.tgt_dico = target_dico

        src_embed, tgt_embed =\
            utils.get_embeds_from_numpy(source_vecs, target_vecs)
        if args.use_cuda:
            self.src_embed = src_embed.cuda()
            self.tgt_embed = tgt_embed.cuda()
        else:
            self.src_embed = src_embed
            self.tgt_embed = tgt_embed

        print('setting models')
        netD = model.netD(self.args)
        netG = model.netG()
        netG.W.weight.data.copy_(torch.diag(torch.ones(300)))
        if args.multi_gpu:
            netD = nn.DataParallel(netD)
            netG = nn.DataParallel(netG)
        if args.use_cuda:
            netD = netD.cuda()
            netG = netG.cuda()
        self.netD = netD
        self.netG = netG
        self.optimizer_D = optim.Adam(self.netD.parameters(),
                                      lr=args.lr,
                                      betas=(args.beta1, 0.999))
        self.optimizer_G = optim.Adam(self.netG.parameters(),
                                      lr=args.lr,
                                      betas=(args.beta1, 0.999))
        self.criterion = nn.BCELoss()
        self.prefix = os.path.basename(args.output_dir)

        self.evaluator = Evaluator(self)

    def train(self):
        args = self.args

        for i_epoch in range(1, args.epoch + 1):

            error_D_list = []
            error_G_list = []

            for niter in tqdm(range(10000)):

                for _ in range(args.dis_step):
                    error_D = self.train_D()
                    error_D_list.append(error_D)

                error_G = self.train_G()

                error_G_list.append(error_G)

                if niter % 500 == 0:
                    print('error_D: ', np.mean(error_D_list))
                    print('error_G: ', np.mean(error_G_list))
                    if args.use_criteria:
                        print('dist cosine mean: ',
                              self.evaluator.dist_mean_cosine())
                        print('caluclating word translation accuracy...')
                        self.evaluator.word_translation()

            result_ = {
                'epoch': i_epoch,
                'error_D': np.mean(error_D_list),
                'error_G': np.mean(error_G_list)
            }
            self.logger.dump(result_)
            if i_epoch % args.log_inter == 0:
                progress_path = os.path.join(args.output_dir, 'progress.json')
                imgpaths = slack_utils.output_progress(progress_path,
                                                       args.output_dir,
                                                       self.prefix)
                if args.slack_output:
                    for imgpath in imgpaths:
                        slack_utils.send_slack_img(imgpath)

    def get_batch_for_disc(self, volatile):
        args = self.args
        batch_size = args.batch_size
        src_ids = torch.LongTensor(batch_size).random_(75000)
        tgt_ids = torch.LongTensor(batch_size).random_(75000)
        if args.use_cuda:
            src_ids = src_ids.cuda()
            tgt_ids = tgt_ids.cuda()
        src_embed = self.src_embed(Variable(src_ids, volatile=True))
        tgt_embed = self.tgt_embed(Variable(tgt_ids, volatile=True))
        src_embed = self.netG(Variable(src_embed.data, volatile=volatile))
        tgt_embed = Variable(tgt_embed.data, volatile=volatile)

        x = torch.cat([src_embed, tgt_embed], 0)
        y = torch.FloatTensor(2 * batch_size).zero_()
        y[:batch_size] = 1 - 0.1
        y[batch_size:] = 0.1
        y = Variable(y.cuda() if args.use_cuda else y)
        return x, y

    def train_D(self):
        self.netD.train()
        self.netG.eval()

        x, y = self.get_batch_for_disc(volatile=True)
        preds = self.netD(Variable(x.data))
        loss = self.criterion(preds, y)
        self.optimizer_D.zero_grad()
        loss.backward()
        self.optimizer_D.step()
        return loss.data[0]

    def train_G(self):
        self.netD.eval()
        self.netG.train()

        x, y = self.get_batch_for_disc(volatile=False)
        preds = self.netD(x)
        loss = self.criterion(preds, 1 - y)

        self.optimizer_G.zero_grad()
        loss.backward()
        self.optimizer_G.step()
        self.orthogonalize()

        return loss.data[0]

    def build_dictionary(self):
        src_emb = self.netG(self.src_embed.weight).data
        tgt_emb = self.tgt_embed.weight.data
        src_emb = src_emb / src_emb.norm(2, 1, keepdim=True).expand_as(src_emb)
        tgt_emb = tgt_emb / tgt_emb.norm(2, 1, keepdim=True).expand_as(tgt_emb)
        self.dico = build_dictionary(src_emb, tgt_emb, self.args)

    def orthogonalize(self):
        if self.args.map_beta > 0:
            W = self.netG.W.weight.data
            beta = self.args.map_beta
            W.copy_((1 + beta) * W - beta * W.mm(W.transpose(0, 1).mm(W)))

    def save_netG_state(self):
        multi_gpu = self.arg.multi_gpus
        odir = self.args.output_dir
        fpath = os.path.join(odir, 'netG_state.pth')
        self.logger.log('saving netG state to', fpath)
        if multi_gpu:
            state_dict = self.netG.module.state_dict()
        else:
            state_dict = self.netG.state_dict()
        torch.save(state_dict, fpath)
Ejemplo n.º 36
0
 def test_eval_indexes_5x5x5(self):
     evaluator = Evaluator(5)
     self.assertEqual(len(evaluator.eval_indexes), 2)
     self.assertEqual(evaluator.eval_indexes[0], (1, 4))
     self.assertEqual(evaluator.eval_indexes[1], (0, 5))
Ejemplo n.º 37
0
 def test_eval_indexes_3x3x3(self):
     evaluator = Evaluator(3)
     self.assertEqual(len(evaluator.eval_indexes), 1)
     self.assertEqual(evaluator.eval_indexes[0], (0, 3))
Ejemplo n.º 38
0
def test(opt, test_loader, testext_loader, net, split):
    start_time = time.time()
    eva = Evaluator(opt.n_classes, opt.bg_err)
    eva_crf = Evaluator(opt.n_classes, opt.bg_err)
    ims = []
    labels = []

    net = net.eval()

    for iteration, batch in enumerate(test_loader):
        im, label = batch
        im = im.cuda()
        label = label.cuda()
        out = net(im)
        prob = F.softmax(out, dim=1)
        for i in range(opt.batch_size):
            prob_np = prob[i].detach().cpu().numpy()
            label_np = label[i].cpu().numpy()
            im_np = im[i].cpu().numpy()
            ims.append(to_image(im[i, :3, :, :]))
            labels.append(label_np)
            eva.register(label_np, prob_np)
            prob_crf = crf(prob_np, im_np, opt.sdims, opt.schan, opt.compat,
                           opt.iters)
            eva_crf.register(label_np, prob_crf)
        print('test',
              str(iteration * opt.batch_size + i).zfill(2),
              time.time() - start_time, 'seconds')

    for iteration, batch in enumerate(testext_loader):
        im, label = batch
        im = im.cuda()
        label = label.cuda()
        out = net(im)
        prob = F.softmax(out, dim=1)
        for i in range(opt.batch_size):
            prob_np = prob[i].detach().cpu().numpy()
            label_np = label[i].cpu().numpy()
            im_np = im[i].cpu().numpy()
            ims.append(to_image(im[i, :3, :, :]))
            labels.append(label_np)
            eva.register(label_np, prob_np)
            prob_crf = crf(prob_np, im_np, opt.sdims, opt.schan, opt.compat,
                           opt.iters)
            eva_crf.register(label_np, prob_crf)
        print('testext',
              str(iteration * opt.batch_size + i).zfill(2),
              time.time() - start_time, 'seconds')

    msa, preds_msa, miou, miiou, preds_miou = eva.evaluate()
    msa_crf, preds_msa_crf, miou_crf, miiou_crf, preds_miou_crf = eva_crf.evaluate(
    )
    print('Pre-CRF:  MSA: {}   mIoU: {}   miIoU: {}'.format(
        round(msa * 100, 1), round(miou * 100, 1), round(miiou * 100, 1)))
    print('Post-CRF: MSA: {}   mIoU: {}   miIoU: {}'.format(
        round(msa_crf * 100, 1), round(miou_crf * 100, 1),
        round(miiou_crf * 100, 1)))
    for i, label in enumerate(labels):
        pred_msa = preds_msa[i]
        pred_msa_crf = preds_msa_crf[i]
        pred_miou = preds_miou[i]
        pred_miou_crf = preds_miou_crf[i]
        vis_im = ims[i]
        vis_label = colormap(label)
        vis_pred_msa = colormap(pred_msa)
        vis_pred_msa_crf = colormap(pred_msa_crf)
        vis_pred_miou = colormap(pred_miou)
        vis_pred_miou_crf = colormap(pred_miou_crf)
        vis_all = np.concatenate(
            (np.concatenate((vis_im, vis_label), axis=2),
             np.concatenate((vis_pred_miou, vis_pred_miou_crf), axis=2)),
            axis=1)
        vis_all = vis_all.transpose((1, 2, 0))
        io.imsave(
            Path(opt.out_path) / split / (str(i).zfill(2) + '.png'), vis_all)
    return msa, miou, miiou, msa_crf, miou_crf, miiou_crf
Ejemplo n.º 39
0
        evaluations.append(id + ' ' + str(score))

    foutput = open(fnoutput, 'w')
    for eval in evaluations:
        foutput.write(eval + '\n')
    foutput.close()


#patterns = [[1, -1, 1, -1, 1]]
#patterns = [[-1, 1, -1, 1, -1, 1, -1, 1, -1, 1],[1, -1, 1, -1, 1, -1, 1, -1, 1, -1]]

patterns = [[-1, 1, -1, 1, -1, 1, -1, 1, -1, 1]]
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1,
#             -1, 1, -1, 1, -1, 1, -1, 1, -1, 1]]
evaluator = Evaluator(patterns)
while True:
    #for i in xrange(2001):
    evaluateBatch()

#patterns = [[-1, 1, -1, 1, -1, 1, -1, 1, -1, 1],[1, -1, 1, -1, 1, -1, 1, -1, 1, -1]]
#evaluator = Evaluator(patterns)
#for i in xrange(1501):
#	evaluateBatch()
Ejemplo n.º 40
0
    def train(self, src_dico, tgt_dico, src_emb, tgt_emb, seed, stage):
        params = self.params
        # Load data
        if not os.path.exists(params.data_dir):
            print("Data path doesn't exists: %s" % params.data_dir)
        if not os.path.exists(self.tune_dir):
            os.makedirs(self.tune_dir)
        if not os.path.exists(self.tune_best_dir):
            os.makedirs(self.tune_best_dir)

        src_word2id = src_dico[1]
        tgt_word2id = tgt_dico[1]

        en = src_emb
        it = tgt_emb

        params = _get_eval_params(params)
        self.params = params
        eval = Evaluator(params, en, it, torch.cuda.is_available())

        AE_optimizer = optim.SGD(filter(
            lambda p: p.requires_grad,
            list(self.X_AE.parameters()) + list(self.Y_AE.parameters())),
                                 lr=params.g_learning_rate)
        # AE_optimizer = optim.SGD(G_params, lr=0.1, momentum=0.9)
        # AE_optimizer = optim.Adam(G_params, lr=params.g_learning_rate, betas=(0.9, 0.9))
        # AE_optimizer = optim.RMSprop(filter(lambda p: p.requires_grad, list(self.X_AE.parameters()) + list(self.Y_AE.parameters())),lr=params.g_learning_rate,alpha=0.9)
        D_optimizer = optim.SGD(list(self.D.parameters()),
                                lr=params.d_learning_rate)
        # D_optimizer = optim.Adam(D_params, lr=params.d_learning_rate, betas=(0.5, 0.9))
        # D_optimizer = optim.RMSprop(list(self.D_X.parameters()) + list(self.D_Y.parameters()), lr=params.d_learning_rate , alpha=0.9)

        # true_dict = get_true_dict(params.data_dir)
        D_acc_epochs = []
        d_loss_epochs = []
        G_AB_loss_epochs = []
        G_BA_loss_epochs = []
        G_AB_recon_epochs = []
        G_BA_recon_epochs = []
        g_loss_epochs = []
        acc_epochs = []

        csls_epochs = []
        best_valid_metric = -100

        # logs for plotting later
        log_file = open(
            "log_src_tgt.txt",
            "w")  # Being overwritten in every loop, not really required
        log_file.write(
            "epoch, disA_loss, disB_loss , disA_acc, disB_acc, g_AB_loss, g_BA_loss, g_AB_recon, g_BA_recon, CSLS, trans_Acc\n"
        )

        if stage == 1:
            self.params.num_epochs = 50
        if stage == 2:
            self.params.num_epochs = 10

        try:
            for epoch in range(self.params.num_epochs):

                G_AB_recon = []
                G_BA_recon = []
                G_X_loss = []
                G_Y_loss = []
                d_losses = []
                g_losses = []
                hit_A = 0
                total = 0
                start_time = timer()
                # lowest_loss = 1e5
                label_D = to_variable(
                    torch.FloatTensor(2 * params.mini_batch_size).zero_())
                label_D[:params.mini_batch_size] = 1 - params.smoothing
                label_D[params.mini_batch_size:] = params.smoothing

                label_G = to_variable(
                    torch.FloatTensor(params.mini_batch_size).zero_())
                label_G = label_G + 1 - params.smoothing
                label_G2 = to_variable(
                    torch.FloatTensor(
                        params.mini_batch_size).zero_()) + params.smoothing

                for mini_batch in range(
                        0, params.iters_in_epoch // params.mini_batch_size):
                    for d_index in range(params.d_steps):
                        D_optimizer.zero_grad()  # Reset the gradients
                        self.D.train()

                        view_X, view_Y = self.get_batch_data_fast_new(en, it)

                        # Discriminator X
                        _, Y_Z = self.Y_AE(view_Y)
                        _, X_Z = self.X_AE(view_X)
                        Y_Z = Y_Z.detach()
                        X_Z = X_Z.detach()
                        input = torch.cat([Y_Z, X_Z], 0)

                        pred = self.D(input)
                        D_loss = self.loss_fn(pred, label_D)
                        D_loss.backward(
                        )  # compute/store gradients, but don't change params
                        d_losses.append(to_numpy(D_loss.data))

                        discriminator_decision_A = to_numpy(pred.data)
                        hit_A += np.sum(
                            discriminator_decision_A[:params.mini_batch_size]
                            >= 0.5)
                        hit_A += np.sum(
                            discriminator_decision_A[params.mini_batch_size:] <
                            0.5)

                        D_optimizer.step(
                        )  # Only optimizes D's parameters; changes based on stored gradients from backward()

                        # Clip weights
                        _clip(self.D, params.clip_value)

                        sys.stdout.write(
                            "[%d/%d] :: Discriminator Loss: %.3f \r" %
                            (mini_batch,
                             params.iters_in_epoch // params.mini_batch_size,
                             np.asscalar(np.mean(d_losses))))
                        sys.stdout.flush()

                    total += 2 * params.mini_batch_size * params.d_steps

                    for g_index in range(params.g_steps):
                        # 2. Train G on D's response (but DO NOT train D on these labels)
                        AE_optimizer.zero_grad()
                        self.D.eval()

                        view_X, view_Y = self.get_batch_data_fast_new(en, it)

                        # Generator X_AE
                        ## adversarial loss
                        X_recon, X_Z = self.X_AE(view_X)
                        Y_recon, Y_Z = self.Y_AE(view_Y)

                        # input = torch.cat([Y_Z, X_Z], 0)

                        predx = self.D(X_Z)
                        D_X_loss = self.loss_fn(predx, label_G)
                        predy = self.D(Y_Z)
                        D_Y_loss = self.loss_fn(predy, label_G2)

                        L_recon_X = 1.0 - torch.mean(
                            self.loss_fn2(view_X, X_recon))
                        L_recon_Y = 1.0 - torch.mean(
                            self.loss_fn2(view_Y, Y_recon))

                        G_loss = D_X_loss + D_Y_loss + L_recon_X + L_recon_Y

                        G_loss.backward()

                        g_losses.append(to_numpy(G_loss.data))
                        G_X_loss.append(
                            to_numpy(D_X_loss.data + L_recon_X.data))
                        G_Y_loss.append(
                            to_numpy(D_Y_loss.data + L_recon_Y.data))
                        G_AB_recon.append(to_numpy(L_recon_X.data))
                        G_BA_recon.append(to_numpy(L_recon_Y.data))

                        AE_optimizer.step()  # Only optimizes G's parameters

                        sys.stdout.write(
                            "[%d/%d] ::                                     Generator Loss: %.3f Generator Y recon: %.3f\r"
                            % (mini_batch,
                               params.iters_in_epoch // params.mini_batch_size,
                               np.asscalar(np.mean(g_losses)),
                               np.asscalar(np.mean(G_BA_recon))))
                        sys.stdout.flush()
                '''for each epoch'''

                D_acc_epochs.append(hit_A / total)
                G_AB_recon_epochs.append(np.asscalar(np.mean(G_AB_recon)))
                G_BA_recon_epochs.append(np.asscalar(np.mean(G_BA_recon)))
                d_loss_epochs.append(np.asscalar(np.mean(d_losses)))
                g_loss_epochs.append(np.asscalar(np.mean(g_losses)))

                print(
                    "Epoch {} : Discriminator Loss: {:.3f}, Discriminator Accuracy: {:.3f}, Generator Loss: {:.3f}, Time elapsed {:.2f} mins"
                    .format(epoch, np.asscalar(np.mean(d_losses)),
                            hit_A / total, np.asscalar(np.mean(g_losses)),
                            (timer() - start_time) / 60))

                if (epoch + 1) % params.print_every == 0:
                    # No need for discriminator weights

                    _, X_Z = self.X_AE(Variable(en))
                    _, Y_Z = self.Y_AE(Variable(it))
                    X_Z = X_Z.data
                    Y_Z = Y_Z.data

                    mstart_time = timer()
                    for method in [params.eval_method]:
                        results = get_word_translation_accuracy(
                            params.src_lang,
                            src_word2id,
                            X_Z,
                            params.tgt_lang,
                            tgt_word2id,
                            Y_Z,
                            method=method,
                            dico_eval='default')
                        acc1 = results[0][1]

                    print('{} takes {:.2f}s'.format(method,
                                                    timer() - mstart_time))
                    print('Method:{} score:{:.4f}'.format(method, acc1))

                    csls = eval.dist_mean_cosine(X_Z, Y_Z)

                    if csls > best_valid_metric:
                        print("New csls value: {}".format(csls))
                        best_valid_metric = csls
                        fp = open(
                            self.tune_best_dir +
                            "/seed_{}_dico_{}_stage_{}_epoch_{}_acc_{:.3f}.tmp"
                            .format(seed, params.dico_build, stage, epoch,
                                    acc1), 'w')
                        fp.close()
                        torch.save(
                            self.X_AE.state_dict(), self.tune_best_dir +
                            '/seed_{}_dico_{}_stage_{}_best_X.t7'.format(
                                seed, params.dico_build, stage))
                        torch.save(
                            self.Y_AE.state_dict(), self.tune_best_dir +
                            '/seed_{}_dico_{}_stage_{}_best_Y.t7'.format(
                                seed, params.dico_build, stage))
                        torch.save(
                            self.D.state_dict(), self.tune_best_dir +
                            '/seed_{}_dico_{}_stage_{}_best_D.t7'.format(
                                seed, params.dico_build, stage))

                    # Saving generator weights
                    fp = open(
                        self.tune_dir +
                        "/seed_{}_stage_{}_epoch_{}_acc_{:.3f}.tmp".format(
                            seed, stage, epoch, acc1), 'w')
                    fp.close()

                    acc_epochs.append(acc1)
                    csls_epochs.append(csls)

            csls_fb, epoch_fb = max([
                (score, index) for index, score in enumerate(csls_epochs)
            ])
            fp = open(
                self.tune_best_dir +
                "/seed_{}_dico_{}_stage_{}_epoch_{}_Acc_{:.3f}_{:.3f}.cslsfb".
                format(seed, params.dico_build, stage, epoch_fb,
                       acc_epochs[epoch_fb], csls_fb), 'w')
            fp.close()

            # Save the plot for discriminator accuracy and generator loss
            # fig = plt.figure()
            # plt.plot(range(0, len(D_A_acc_epochs)), D_A_acc_epochs, color='b', label='D_A')
            # plt.plot(range(0, len(D_B_acc_epochs)), D_B_acc_epochs, color='r', label='D_B')
            # plt.ylabel('D_accuracy')
            # plt.xlabel('epochs')
            # plt.legend()
            # fig.savefig(self.tune_dir + '/seed_{}_stage_{}_D_acc.png'.format(seed, stage))
            #
            # fig = plt.figure()
            # plt.plot(range(0, len(D_A_loss_epochs)), D_A_loss_epochs, color='b', label='D_A')
            # plt.plot(range(0, len(D_B_loss_epochs)), D_B_loss_epochs, color='r', label='D_B')
            # plt.ylabel('D_losses')
            # plt.xlabel('epochs')
            # plt.legend()
            # fig.savefig(self.tune_dir + '/seed_{}_stage_{}_D_loss.png'.format(seed, stage))
            #
            # fig = plt.figure()
            # plt.plot(range(0, len(G_AB_loss_epochs)), G_AB_loss_epochs, color='b', label='G_AB')
            # plt.plot(range(0, len(G_BA_loss_epochs)), G_BA_loss_epochs, color='r', label='G_BA')
            # plt.ylabel('G_losses')
            # plt.xlabel('epochs')
            # plt.legend()
            # fig.savefig(self.tune_dir + '/seed_{}_stage_{}_G_loss.png'.format(seed,stage))
            #
            # fig = plt.figure()
            # plt.plot(range(0, len(G_AB_recon_epochs)), G_AB_recon_epochs, color='b', label='G_AB')
            # plt.plot(range(0, len(G_BA_recon_epochs)), G_BA_recon_epochs, color='r', label='G_BA')
            # plt.ylabel('G_recon_loss')
            # plt.xlabel('epochs')
            # plt.legend()
            # fig.savefig(self.tune_dir + '/seed_{}_stage_{}_G_Recon.png'.format(seed,stage))

            # fig = plt.figure()
            # plt.plot(range(0, len(L_Z_loss_epoches)), L_Z_loss_epoches, color='b', label='L_Z')
            # plt.ylabel('L_Z_loss')
            # plt.xlabel('epochs')
            # plt.legend()
            # fig.savefig(tune_dir + '/seed_{}_stage_{}_L_Z.png'.format(seed,stage))

            fig = plt.figure()
            plt.plot(range(0, len(acc_epochs)),
                     acc_epochs,
                     color='b',
                     label='trans_acc1')
            plt.ylabel('trans_acc')
            plt.xlabel('epochs')
            plt.legend()
            fig.savefig(self.tune_dir +
                        '/seed_{}_stage_{}_trans_acc.png'.format(seed, stage))

            fig = plt.figure()
            plt.plot(range(0, len(csls_epochs)),
                     csls_epochs,
                     color='b',
                     label='csls')
            plt.ylabel('csls')
            plt.xlabel('epochs')
            plt.legend()
            fig.savefig(self.tune_dir +
                        '/seed_{}_stage_{}_csls.png'.format(seed, stage))

            fig = plt.figure()
            plt.plot(range(0, len(g_loss_epochs)),
                     g_loss_epochs,
                     color='b',
                     label='G_loss')
            plt.ylabel('g_loss')
            plt.xlabel('epochs')
            plt.legend()
            fig.savefig(self.tune_dir +
                        '/seed_{}_g_stage_{}_loss.png'.format(seed, stage))

            fig = plt.figure()
            plt.plot(range(0, len(d_loss_epochs)),
                     d_loss_epochs,
                     color='b',
                     label='csls')
            plt.ylabel('D_loss')
            plt.xlabel('epochs')
            plt.legend()
            fig.savefig(self.tune_dir +
                        '/seed_{}_stage_{}_d_loss.png'.format(seed, stage))
            plt.close('all')

        except KeyboardInterrupt:
            print("Interrupted.. saving model !!!")
            torch.save(self.X_AE.state_dict(), 'X_model_interrupt.t7')
            torch.save(self.Y_AE.state_dict(), 'Y_model_interrupt.t7')
            torch.save(self.D.state_dict(), 'd_model_interrupt.t7')
            log_file.close()
            exit()

        log_file.close()
        return
Ejemplo n.º 41
0
 def ies_normal_accuracy_scores(self):
     self.logger.println("normal accuracy scores ies called")
     evaluator = Evaluator()
     evaluator.get_ies_scores()
Ejemplo n.º 42
0
    vocab = load_words(train_exs, cutoff=5)

    device = torch.device("cuda" if args.cuda else "cpu")

    train_dataset = dataset.WikiTableDataset(train_exs, vocab)
    train_sampler = torch.utils.data.sampler.RandomSampler(train_dataset)
    dev_dataset = dataset.WikiTableDataset(dev_exs, vocab)
    dev_sampler = torch.utils.data.sampler.SequentialSampler(dev_dataset)
    dev_loader = torch.utils.data.DataLoader(dev_dataset,
                                             batch_size=1,
                                             sampler=dev_sampler,
                                             num_workers=args.data_workers,
                                             collate_fn=dataset.batchify,
                                             pin_memory=args.cuda)

    evaluator = Evaluator("../tables/tagged/", "../tables/db/",
                          "../stanford-corenlp-full-2018-10-05/")

    if args.test:
        test_exs = em_process(load_dataset(args.test_file))
        test_dataset = dataset.WikiTableDataset(test_exs, vocab)
        test_sampler = torch.utils.data.sampler.SequentialSampler(test_dataset)
        test_loader = torch.utils.data.DataLoader(
            test_dataset,
            batch_size=1,
            sampler=test_sampler,
            num_workers=args.data_workers,
            collate_fn=dataset.batchify,
            pin_memory=args.cuda)

        if not args.cuda:
            model = torch.load(args.load_model, map_location={'cuda:0': 'cpu'})
Ejemplo n.º 43
0
def main():
    parser = argparse.ArgumentParser(description="sentence Hi_LSTM-attent-pooling model")
    parser.add_argument('--train_flag', action='store_true', help='Train or eval')
    parser.add_argument('--fine_tune', action='store_true', help='Fine tune word embeddings')
    parser.add_argument('--embedding', type=str, default='word2vec', help='Word embedding type, word2vec, senna or glove')
    parser.add_argument('--embedding_dict', type=str, default=None, help='Pretrained embedding path')
    parser.add_argument('--embedding_dim', type=int, default=64, help='Only useful when embedding is randomly initialised')

    parser.add_argument('--use_char', action='store_true', help='Whether use char embedding or not')
    parser.add_argument('--num_epochs', type=int, default=20, help='number of epochs for training')
    parser.add_argument('--batch_size', type=int, default=10, help='Number of texts in each batch')
    parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='<int>', default=4000, help="Vocab size (default=4000)")

    parser.add_argument('--rnn_type', type=str, default='LSTM', help='Recurrent type')
    parser.add_argument('--lstm_units', type=int, default=100, help='Num of hidden units in recurrent layer')

    # parser.add_argument('--project_hiddensize', type=int, default=100, help='num of units in projection layer')
    parser.add_argument('--optimizer', choices=['sgd', 'momentum', 'nesterov', 'adagrad', 'rmsprop'], help='updating algorithm', default='sgd')
    parser.add_argument('--learning_rate', type=float, default=0.1, help='Initial learning rate')
    parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate for layers')
    parser.add_argument('--oov', choices=['random', 'embedding'], help="Embedding for oov word", required=True)
    parser.add_argument('--l2_value', type=float, default=0.001, help='l2 regularizer value')
    parser.add_argument('--checkpoint_path', type=str, help='checkpoint directory')

    parser.add_argument('--train')  # "data/word-level/*.train"
    parser.add_argument('--dev')
    parser.add_argument('--test')
    parser.add_argument('--prompt_id', type=int, default=1, help='prompt id of essay set')
    parser.add_argument('--bidirectional', action='store_true', help='bidirectional RNN or not')
    parser.add_argument('--masked', action='store_true', help='using mask')

    # to support init bias of output layer of the network
    parser.add_argument('--init_bias', action='store_true', help='initial bias of output' + 
                        'layer with the mean score of training data')

    args = parser.parse_args()

    train_flag = args.train_flag
    fine_tune = args.fine_tune
    USE_CHAR = args.use_char

    batch_size = args.batch_size
    checkpoint_dir = args.checkpoint_path
    num_epochs = args.num_epochs

    bidirectional = args.bidirectional
    if bidirectional:
        modelname = "sent_hibilstm-attent.prompt%s.%sunits.bs%s.hdf5" % (args.prompt_id, args.lstm_units, batch_size)
        imgname = "sent_hibilstm-attent.prompt%s.%sunits.bs%s.png" % (args.prompt_id, args.lstm_units, batch_size)
    else:
        modelname = "sent_hilstm-attent.prompt%s.%sunits.bs%s.hdf5" % (args.prompt_id, args.lstm_units, batch_size)
        imgname = "sent_hilstm-attent.prompt%s.%sunits.bs%s.png" % (args.prompt_id, args.lstm_units, batch_size)

    if args.masked:
        modelname = 'masked_' + modelname
        imgname = 'masked_' + imgname

    if USE_CHAR:
        modelname = 'char_' + modelname
        imgname = 'char_' + imgname
    modelpath = os.path.join(checkpoint_dir, modelname)
    imgpath = os.path.join(checkpoint_dir, imgname)

    datapaths = [args.train, args.dev, args.test]
    embedding_path = args.embedding_dict
    oov = args.oov
    embedding = args.embedding
    embedd_dim = args.embedding_dim
    prompt_id = args.prompt_id

    (X_train, Y_train, mask_train), (X_dev, Y_dev, mask_dev), (X_test, Y_test, mask_test), \
            vocab, vocab_size, embed_table, overal_maxlen, overal_maxnum, init_mean_value = prepare_sentence_data(datapaths, \
            embedding_path, embedding, embedd_dim, prompt_id, args.vocab_size, tokenize_text=True, \
            to_lower=True, sort_by_len=False, vocab_path=None, score_index=6)

    # print type(embed_table)
    if embed_table is not None:
        embedd_dim = embed_table.shape[1]
        embed_table = [embed_table]
        
    max_sentnum = overal_maxnum
    max_sentlen = overal_maxlen
    # print embed_table
    # print X_train[0, 0:10, :]
    # print Y_train[0:10]

    X_train = X_train.reshape((X_train.shape[0], X_train.shape[1]*X_train.shape[2]))
    X_dev = X_dev.reshape((X_dev.shape[0], X_dev.shape[1]*X_dev.shape[2]))
    X_test = X_test.reshape((X_test.shape[0], X_test.shape[1]*X_test.shape[2]))

    logger.info("X_train shape: %s" % str(X_train.shape))
    if USE_CHAR:
        raise NotImplementedError

    else:
        if args.masked:
            model = masked_hilstm.build_model(args, vocab_size, max_sentnum, max_sentlen, embedd_dim, embed_table, True, init_mean_value)
        else:
            model = build_model(args, vocab_size, max_sentnum, max_sentlen, embedd_dim, embed_table, True, init_mean_value)

    # TODO here, modified to evaluate every epoch
    evl = Evaluator(args.prompt_id, False, checkpoint_dir, modelname, X_train, X_dev, X_test, None, None, None, Y_train, Y_dev, Y_test, False)
    
    # Initial evaluation
    logger.info("Initial evaluation: ")
    evl.evaluate(model, -1, print_info=True)
    logger.info("Train model")
    for ii in xrange(args.num_epochs):
        logger.info('Epoch %s/%s' % (str(ii+1), args.num_epochs))
        start_time = time.time()
        model.fit(X_train, Y_train, batch_size=args.batch_size, nb_epoch=1, verbose=0, shuffle=True)
        tt_time = time.time() - start_time
        logger.info("Training one epoch in %.3f s" % tt_time)
        evl.evaluate(model, ii+1)
        evl.print_info()

    evl.print_final_info()
Ejemplo n.º 44
0
    # dataset = BatchData(PrefetchData(train_set, 4, 4), BATCH_SIZE)

    lr_schedule = [(80, 1e-4), (120, 1e-5)]
    # lr_schedule = [(i, 5e-5) for i in range(260)]
    # get the config which contains everything necessary in a training
    config = AutoResumeTrainConfig(
        always_resume=False,
        model=Model(),
        # The input source for training. FeedInput is slow, this is just for demo purpose.
        # In practice it's best to use QueueInput or others. See tutorials for details.
        data=QueueInput(
            BatchData2Biggest(
                PrefetchData(train_set,
                             multiprocessing.cpu_count() // 2,
                             multiprocessing.cpu_count() // 2), BATCH_SIZE)),
        # starting_epoch=60,
        callbacks=[
            ModelSaver(),  # save the model after every epoch
            ScheduledHyperParamSetter('learning_rate', lr_schedule),
            # compute mAP on val set
            Evaluator('/media/neil/DATA/mysunrgbd',
                      'training',
                      1,
                      idx_list=list(range(1, 5051))),
            # MaxSaver('val_accuracy'),  # save the model with highest accuracy
        ],
        # steps_per_epoch=100,
        max_epoch=260,
    )
    launch_train_with_config(config, SimpleTrainer())
Ejemplo n.º 45
0
 def test_give_a_note_on_random1(self):
     cube = Cube('random1.txt')
     evaluator = Evaluator(3)
     self.assertEqual(evaluator.give_a_note(cube), (18,))
Ejemplo n.º 46
0
class SupervisedTrainer(object):
    """ The SupervisedTrainer class helps in setting up a training framework in a
    supervised setting.

    Args:
        expt_dir (optional, str): experiment Directory to store details of the experiment,
            by default it makes a folder in the current directory to store the details (default: `experiment`).
        loss (seq2seq.loss.loss.Loss, optional): loss for training, (default: seq2seq.loss.NLLLoss)
        batch_size (int, optional): batch size for experiment, (default: 64)
        checkpoint_every (int, optional): number of epochs to checkpoint after, (default: 100)
    """
    def __init__(self,
                 expt_dir='experiment',
                 loss=NLLLoss(),
                 batch_size=64,
                 random_seed=None,
                 checkpoint_every=100,
                 print_every=100):
        self._trainer = "Simple Trainer"
        self.random_seed = random_seed
        if random_seed is not None:
            random.seed(random_seed)
            torch.manual_seed(random_seed)
        self.loss = loss
        self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size)
        self.optimizer = None
        self.checkpoint_every = checkpoint_every
        self.print_every = print_every

        if not os.path.isabs(expt_dir):
            expt_dir = os.path.join(os.getcwd(), expt_dir)
        self.expt_dir = expt_dir
        if not os.path.exists(self.expt_dir):
            os.makedirs(self.expt_dir)
        self.batch_size = batch_size

        self.logger = logging.getLogger(__name__)

    def _train_batch(self, input_variable, input_lengths, target_variable,
                     model, teacher_forcing_ratio):
        loss = self.loss
        # Forward propagation
        decoder_outputs, decoder_hidden, other = model(
            input_variable,
            input_lengths,
            target_variable,
            teacher_forcing_ratio=teacher_forcing_ratio)
        # Get loss
        loss.reset()
        for step, step_output in enumerate(decoder_outputs):
            batch_size = target_variable.size(0)
            loss.eval_batch(step_output.contiguous().view(batch_size, -1),
                            target_variable[:, step + 1])
        # Backward propagation
        model.zero_grad()
        loss.backward()
        self.optimizer.step()

        return loss.get_loss()

    def _train_epoches(self,
                       data,
                       model,
                       n_epochs,
                       start_epoch,
                       start_step,
                       dev_data=None,
                       teacher_forcing_ratio=0):
        log = self.logger

        print_loss_total = 0  # Reset every print_every
        epoch_loss_total = 0  # Reset every epoch

        device = None if torch.cuda.is_available() else -1
        batch_iterator = torchtext.data.BucketIterator(
            dataset=data,
            batch_size=self.batch_size,
            sort=False,
            sort_within_batch=True,
            sort_key=lambda x: len(x.src),
            device=device,
            repeat=False)

        steps_per_epoch = len(batch_iterator)
        total_steps = steps_per_epoch * n_epochs

        step = start_step
        step_elapsed = 0

        best_loss = None
        save_flag = False

        for epoch in range(start_epoch, n_epochs + 1):
            log.debug("Epoch: %d, Step: %d" % (epoch, step))

            batch_generator = batch_iterator.__iter__()
            # consuming seen batches from previous training
            for _ in range((epoch - 1) * steps_per_epoch, step):
                next(batch_generator)

            model.train(True)
            for batch in batch_generator:
                step += 1
                step_elapsed += 1

                input_variables, input_lengths = getattr(
                    batch, GlobalNames.src_field_name)
                target_variables = getattr(batch, GlobalNames.tgt_field_name)

                loss = self._train_batch(input_variables,
                                         input_lengths.tolist(),
                                         target_variables, model,
                                         teacher_forcing_ratio)

                # Record average loss
                print_loss_total += loss
                epoch_loss_total += loss

                if step % self.print_every == 0 and step_elapsed > self.print_every:
                    print_loss_avg = print_loss_total / self.print_every
                    print_loss_total = 0
                    log_msg = 'Progress: %d%%, Train %s: %.4f' % (
                        step / total_steps * 100, self.loss.name,
                        print_loss_avg)
                    log.info(log_msg)

            if step_elapsed == 0: continue

            epoch_loss_avg = epoch_loss_total / min(steps_per_epoch,
                                                    step - start_step)
            epoch_loss_total = 0
            log_msg = "Finished epoch %d: Train %s: %.4f" % (
                epoch, self.loss.name, epoch_loss_avg)
            if dev_data is not None:
                dev_loss, accuracy = self.evaluator.evaluate2(model, dev_data)
                self.optimizer.update(dev_loss, epoch)
                log_msg += ", Dev %s: %.4f, Accuracy: %.4f" % (
                    self.loss.name, dev_loss, accuracy)
                model.train(mode=True)
                if best_loss is not None:
                    if dev_loss < best_loss:
                        save_flag = True
                else:
                    best_loss = dev_loss
                    save_flag = True
            else:
                save_flag = True
                self.optimizer.update(epoch_loss_avg, epoch)

            if save_flag:
                Checkpoint(
                    model=model,
                    optimizer=self.optimizer,
                    epoch=epoch,
                    step=step,
                    input_vocab=data.fields[GlobalNames.src_field_name].vocab,
                    output_vocab=data.fields[
                        GlobalNames.tgt_field_name].vocab).save(self.expt_dir)

            log.info(log_msg)

    def train(self,
              model,
              data,
              num_epochs=5,
              resume=False,
              dev_data=None,
              optimizer=None,
              teacher_forcing_ratio=0):
        """ Run training for a given model.

        Args:
            model (seq2seq.models): model to run training on, if `resume=True`, it would be
               overwritten by the model loaded from the latest checkpoint.
            data (seq2seq.dataset.dataset.Dataset): dataset object to train on
            num_epochs (int, optional): number of epochs to run (default 5)
            resume(bool, optional): resume training with the latest checkpoint, (default False)
            dev_data (seq2seq.dataset.dataset.Dataset, optional): dev Dataset (default None)
            optimizer (seq2seq.optim.Optimizer, optional): optimizer for training
               (default: Optimizer(pytorch.optim.Adam, max_grad_norm=5))
            teacher_forcing_ratio (float, optional): teaching forcing ratio (default 0)
        Returns:
            model (seq2seq.models): trained model.
        """
        # If training is set to resume
        if resume:
            latest_checkpoint_path = Checkpoint.get_latest_checkpoint(
                self.expt_dir)
            resume_checkpoint = Checkpoint.load(latest_checkpoint_path)
            model = resume_checkpoint.model
            self.optimizer = resume_checkpoint.optimizer

            # A walk around to set optimizing parameters properly
            resume_optim = self.optimizer.optimizer
            defaults = resume_optim.param_groups[0]
            defaults.pop('params', None)
            defaults.pop('initial_lr', None)
            self.optimizer.optimizer = resume_optim.__class__(
                model.parameters(), **defaults)

            start_epoch = resume_checkpoint.epoch
            step = resume_checkpoint.step
        else:
            start_epoch = 1
            step = 0
            if optimizer is None:
                optimizer = Optimizer(optim.Adam(model.parameters()),
                                      max_grad_norm=5)
            self.optimizer = optimizer

        self.logger.info("Optimizer: %s, Scheduler: %s" %
                         (self.optimizer.optimizer, self.optimizer.scheduler))

        self._train_epoches(data,
                            model,
                            num_epochs,
                            start_epoch,
                            step,
                            dev_data=dev_data,
                            teacher_forcing_ratio=teacher_forcing_ratio)
        return model
Ejemplo n.º 47
0
 def test_eval_indexes_6x6x6(self):
     evaluator = Evaluator(6)
     self.assertEqual(len(evaluator.eval_indexes), 3)
     self.assertEqual(evaluator.eval_indexes[0], (2, 4))
     self.assertEqual(evaluator.eval_indexes[1], (1, 5))
     self.assertEqual(evaluator.eval_indexes[2], (0, 6))
Ejemplo n.º 48
0
 def run(l):
     best_net = Net(best_net_filename)
     new_net = Net(new_net_filename)
     evaluate = Evaluator(best_net, new_net, num_games=1, num_simulations=num_simulations)
     result = evaluate.start()
     l.append(result)
Ejemplo n.º 49
0
 def test_eval_indexes_4x4x4(self):
     evaluator = Evaluator(4)
     self.assertEqual(len(evaluator.eval_indexes), 2)
     self.assertEqual(evaluator.eval_indexes[0], (1, 3))
     self.assertEqual(evaluator.eval_indexes[1], (0, 4))
from model import DCGAN
from evaluator import Evaluator
import argparse

	
if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('--mode')
	
	args = parser.parse_args()
	
	model = DCGAN(mode=args.mode, learning_rate=0.0002, fx_dim=100, beta=0.5)
	evaluator = Evaluator(model, batch_size=128, update_ratio=2, train_iter=20, cifar_dir='/home/robot/Dataset/cifar10_data/', log_dir='/home/robot/Experiments/DCGAN/logs/', save_model_dir='/home/robot/Experiments/DCGAN/trained_models/', save_sample_dir='/home/robot/Experiments/DCGAN/sampled_images/', test_model='/home/robot/Experiments/DCGAN/trained_models/dcgan-20')
	
	if args.mode == 'train':
		evaluator.train()
	else:
		evaluator.eval()
Ejemplo n.º 51
0
class GVF:
    """Implements General Value Functions.

    General Value Functions pose a question defined by the cumulant, gamma,
    and the target policy, that is learned by a learning algorithm, here called
    the ``learner``.

    Args:
        cumulant (fun): Function of observation that gives a float value.
        gamma (fun): Function of observation that gives a float value. Together
            with cumulant, makes the return that the agent tries to predict.
        target_policy (Policy): Policy under which the agent makes its
            predictions. Can be the same as the behavior policy.
        num_features (int): Number of features that are used.
        alpha0 (float): Value to calculate beta0 for RUPEE.
        alpha (float): Value to calculate alpha for RUPEE.
        name (str): Name of the GVF for recording data.
        learner: Class instance with a ``predict`` and ``update`` function,
            and ``theta``, ``tderr_elig``, and ``delta`` attributes. For
            example, GTD.
        feature_indices (numpy array of bool): Indices of the features to use.
        use_MSRE (bool): Whether or not to calculate MSRE.
    """
    def __init__(self,
                 cumulant,
                 gamma,
                 target_policy,
                 num_features,
                 alpha0,
                 alpha,
                 name,
                 learner,
                 feature_indices,
                 use_MSRE=False,
                 **kwargs):

        self.cumulant = cumulant
        self.gamma = gamma
        self.target_policy = target_policy
        self.last_cumulant = 0.0
        self.phi = np.zeros(num_features)
        self.rho = 1.0
        self.last_prediction = 0.0

        self.name = name
        self.feature_indices = feature_indices
        self.learner = learner
        self.uses_action_state = feature_indices.size < num_features
        self.use_MSRE = use_MSRE

        self.time_step = 0

        # See Adam White's PhD Thesis, section 8.4.2
        alpha_rupee = 5 * alpha
        beta0_rupee = alpha0 / 30
        self.evaluator = Evaluator(gvf_name=name,
                                   num_features=num_features,
                                   alpha_rupee=alpha_rupee,
                                   beta0_rupee=beta0_rupee,
                                   use_MSRE=use_MSRE)

    def predict(self, phi, action=None, **kwargs):
        if self.uses_action_state:
            return self.learner.predict(phi[self.feature_indices], action)
        else:
            return self.learner.predict(phi[self.feature_indices])

    def update(self,
               last_observation,
               phi,
               last_action,
               observation,
               phi_prime,
               mu,
               action):

        self.last_prediction = self.predict(phi_prime, action)

        # update action probabilities and get probability of last action
        self.target_policy.update(phi, last_observation)
        pi = self.target_policy.get_probability(last_action)

        cumulant = self.cumulant(observation)

        # get relevant indices in phi
        phi = phi[self.feature_indices]
        phi_prime = phi_prime[self.feature_indices]

        self.rho = pi / mu
        kwargs = {"phi": phi,
                  "last_action": last_action,
                  "phi_prime": phi_prime,
                  "rho": self.rho,
                  "gamma": self.gamma(observation),
                  "cumulant": cumulant,
                  }

        phi = self.learner.update(**kwargs)

        self.evaluator.update(theta=self.learner.theta,
                              time_step=self.time_step,
                              tderr_elig=self.learner.tderr_elig,
                              delta=self.learner.delta,
                              phi=phi,
                              rho=self.rho)

        self.phi = phi_prime
        self.last_cumulant = cumulant
        self.time_step += 1
Ejemplo n.º 52
0
 def evaluate(self, tokens):
     if '=' in tokens:
         self.evaluate_variable(tokens)
     else:
         postfix = Converter.infix_to_postfix(tokens)
         return Evaluator.eval_rpn(postfix, self.variables)
Ejemplo n.º 53
0
def main():
    parser = argparse.ArgumentParser(description="sentence Hi_CNN_LSTM model")
    parser.add_argument(
        '--embedding',
        type=str,
        default='glove',
        help='Word embedding type, glove, word2vec, senna or random')
    parser.add_argument('--embedding_dict',
                        type=str,
                        default='glove/glove.6B.50d.txt',
                        help='Pretrained embedding path')
    parser.add_argument(
        '--embedding_dim',
        type=int,
        default=50,
        help='Only useful when embedding is randomly initialised')

    parser.add_argument('--num_epochs',
                        type=int,
                        default=50,
                        help='number of epochs for training')
    parser.add_argument('--batch_size',
                        type=int,
                        default=10,
                        help='Number of texts in each batch')
    parser.add_argument("-v",
                        "--vocab-size",
                        dest="vocab_size",
                        type=int,
                        metavar='<int>',
                        default=4000,
                        help="Vocab size (default=4000)")

    parser.add_argument('--nbfilters',
                        type=int,
                        default=100,
                        help='Num of filters in conv layer')
    parser.add_argument('--filter1_len',
                        type=int,
                        default=5,
                        help='filter length in 1st conv layer')
    parser.add_argument('--lstm_units',
                        type=int,
                        default=100,
                        help='Num of hidden units in recurrent layer')

    parser.add_argument('--optimizer',
                        choices=['sgd', 'adagrad', 'rmsprop'],
                        help='Optimizer',
                        default='rmsprop')
    parser.add_argument('--learning_rate',
                        type=float,
                        default=0.001,
                        help='Initial learning rate')
    parser.add_argument('--dropout',
                        type=float,
                        default=0.5,
                        help='Dropout rate for layers')
    parser.add_argument('--l2_value', type=float, help='l2 regularizer value')
    parser.add_argument('--checkpoint_path',
                        type=str,
                        help='checkpoint directory',
                        default='checkpoints')

    parser.add_argument('--train',
                        type=str,
                        help='train file',
                        default='data/fold_0/train.tsv')
    parser.add_argument('--dev',
                        type=str,
                        help='dev file',
                        default='data/fold_0/dev.tsv')
    parser.add_argument('--test',
                        type=str,
                        help='test file',
                        default='data/fold_0/test.tsv')
    parser.add_argument('--prompt_id',
                        type=int,
                        default=3,
                        help='prompt id of essay set')
    parser.add_argument(
        '--init_bias',
        action='store_true',
        help='init the last layer bias with average score of training data')
    parser.add_argument('--mode',
                        type=str,
                        choices=['att', 'co'],
                        default='co',
                        help='attention-pooling, or co-attention pooling')

    args = parser.parse_args()

    batch_size = args.batch_size
    checkpoint_dir = args.checkpoint_path
    num_epochs = args.num_epochs

    modelname = "%s.prompt%s.%sfilters.bs%s" % (args.mode, args.prompt_id,
                                                args.nbfilters, batch_size)

    datapaths = [args.train, args.dev, args.test]
    embedding_path = args.embedding_dict
    embedding = args.embedding
    emb_dim = args.embedding_dim
    prompt_id = args.prompt_id

    mode = args.mode
    need_context = mode in ['co']

    (X_train, Y_train, mask_train, train_context, text_train), \
    (X_dev, Y_dev, mask_dev, dev_context, text_dev), \
    (X_test, Y_test, mask_test, test_context, text_test), \
    vocab, vocab_size, emb_table, overall_maxlen, overall_maxnum, init_mean_value, context_len, context_num = \
        data_prepare.prepare_sentence_data(
            datapaths,
            embedding_path,
            embedding,
            emb_dim,
            prompt_id,
            args.vocab_size,
            tokenize_text=True,
            to_lower=True,
            vocab_path=None,
            score_index=6,
            need_context=need_context
        )

    if emb_table is not None:
        emb_dim = emb_table.shape[1]
        emb_table = [emb_table]

    max_sentnum = overall_maxnum
    max_sentlen = overall_maxlen

    X_train = X_train.reshape(
        (X_train.shape[0], X_train.shape[1] * X_train.shape[2]))
    X_dev = X_dev.reshape((X_dev.shape[0], X_dev.shape[1] * X_dev.shape[2]))
    X_test = X_test.reshape(
        (X_test.shape[0], X_test.shape[1] * X_test.shape[2]))

    train_context = train_context.reshape(
        (train_context.shape[0],
         train_context.shape[1] * train_context.shape[2]))
    dev_context = dev_context.reshape(
        (dev_context.shape[0], dev_context.shape[1] * dev_context.shape[2]))
    test_context = test_context.reshape(
        (test_context.shape[0], test_context.shape[1] * test_context.shape[2]))

    logger.info("X_train shape: %s" % str(X_train.shape))
    logger.info("X_dev shape: %s" % str(X_dev.shape))
    logger.info("X_test shape: %s" % str(X_test.shape))

    if mode == 'att':
        model = build_hrcnn_model(args, vocab_size, max_sentnum, max_sentlen,
                                  emb_dim, emb_table, True, init_mean_value)
        x_train = X_train
        y_train = Y_train
        x_dev = X_dev
        y_dev = Y_dev
        x_test = X_test
        y_test = Y_test
    elif mode == 'co':
        model = build_shrcnn_model(args, vocab_size, max_sentnum, max_sentlen,
                                   context_num, context_len, emb_dim,
                                   emb_table, True, init_mean_value)
        x_train = [X_train, train_context]
        y_train = Y_train
        x_dev = [X_dev, dev_context]
        y_dev = Y_dev
        x_test = [X_test, test_context]
        y_test = Y_test
    else:
        raise NotImplementedError

    evl = Evaluator(prompt_id, checkpoint_dir, modelname, x_train, x_dev,
                    x_test, y_train, y_dev, y_test)

    # Initial evaluation
    logger.info("Initial evaluation: ")
    evl.evaluate(model, -1, print_info=True)
    logger.info("Train model")
    for ii in range(num_epochs):
        logger.info('Epoch %s/%s' % (str(ii + 1), num_epochs))
        start_time = time.time()
        model.fit(x=x_train,
                  y=y_train,
                  batch_size=batch_size,
                  epochs=1,
                  verbose=0,
                  shuffle=True)
        tt_time = time.time() - start_time
        logger.info("Training one epoch in %.3f s" % tt_time)
        evl.evaluate(model, ii + 1, print_info=True)

    evl.print_final_info()
Ejemplo n.º 54
0
 def test_eval_indexes_2x2x2(self):
     evaluator = Evaluator(2)
     self.assertEqual(len(evaluator.eval_indexes), 1)
     self.assertEqual(evaluator.eval_indexes[0], (0, 2))
Ejemplo n.º 55
0
#  Purpose of this module is to test against an opening range - One Raiser in Front

import random
from evaluator import Evaluator
from deck import Deck
from openpyxl import load_workbook
from card import Card
import e_functions as matt_custom

# 1. Initiate class

evaluator = Evaluator()

# 2. Workbooks used for outpu
# t and input

# 2a Input workbook is range_lookup.xlsm
wb_input = load_workbook(filename='combo_method.xlsm', data_only=True)
input_sheet = wb_input['preflop']

# 2b Output workbook
dest_filename = 'combo_method_output.xlsx'
wb_output = load_workbook(filename=dest_filename, read_only=False)
ws1 = wb_output['preflop']

ws_all_combos = wb_input['all_combos']

output_counter = ws1.max_row + 1

hero_combo_count = input_sheet.cell(row=2, column=3).value
Ejemplo n.º 56
0
 def test_give_a_note_on_3x3x3_solved(self):
     cube = Cube('3x3x3_solved.txt')
     evaluator = Evaluator(3)
     with self.assertRaises(SolvedCubeException):
         evaluator.give_a_note(cube)
Ejemplo n.º 57
0
class Trainer(object):
    def __init__(
        self,
        env,
        eval_env=None,
        image_size=(45, 45, 45),
        update_frequency=4,
        replay_buffer_size=1e6,
        init_memory_size=5e4,
        max_episodes=100,
        steps_per_episode=50,
        eps=1,
        min_eps=0.1,
        delta=0.001,
        batch_size=4,
        gamma=0.9,
        number_actions=6,
        frame_history=4,
        model_name="CommNet",
        logger=None,
        train_freq=1,
    ):
        self.env = env
        self.eval_env = eval_env
        self.agents = env.agents
        self.image_size = image_size
        self.update_frequency = update_frequency
        self.replay_buffer_size = replay_buffer_size
        self.init_memory_size = init_memory_size
        self.max_episodes = max_episodes
        self.steps_per_episode = steps_per_episode
        self.eps = eps
        self.min_eps = min_eps
        self.delta = delta
        self.batch_size = batch_size
        self.gamma = gamma
        self.number_actions = number_actions
        self.frame_history = frame_history
        self.epoch_length = self.env.files.num_files
        self.best_val_distance = float('inf')
        self.buffer = ReplayMemory(self.replay_buffer_size, self.image_size,
                                   self.frame_history, self.agents)
        self.dqn = DQN(self.agents,
                       self.frame_history,
                       logger=logger,
                       type=model_name)
        self.dqn.q_network.train(True)
        self.evaluator = Evaluator(eval_env, self.dqn.q_network, logger,
                                   self.agents, steps_per_episode)
        self.logger = logger
        self.train_freq = train_freq

    def train(self):
        self.logger.log(self.dqn.q_network)
        self.set_reproducible()
        self.init_memory()
        episode = 1
        acc_steps = 0
        epoch_distances = []
        while episode <= self.max_episodes:
            # Reset the environment for the start of the episode.
            obs = self.env.reset()
            terminal = [False for _ in range(self.agents)]
            losses = []
            score = [0] * self.agents
            for step_num in range(self.steps_per_episode):
                acc_steps += 1
                acts, q_values = self.get_next_actions(
                    self.buffer.recent_state())
                # Step the agent once, and get the transition tuple
                obs, reward, terminal, info = self.env.step(
                    np.copy(acts), q_values, terminal)
                score = [sum(x) for x in zip(score, reward)]
                self.buffer.append((obs, acts, reward, terminal))
                if acc_steps % self.train_freq == 0:
                    mini_batch = self.buffer.sample(self.batch_size)
                    loss = self.dqn.train_q_network(mini_batch, self.gamma)
                    losses.append(loss)
                if all(t for t in terminal):
                    break
            epoch_distances.append(
                [info['distError_' + str(i)] for i in range(self.agents)])
            self.append_episode_board(info, score, "train", episode)
            if (episode * self.epoch_length) % self.update_frequency == 0:
                self.dqn.copy_to_target_network()
            self.eps = max(self.min_eps, self.eps - self.delta)
            # Every epoch
            if episode % self.epoch_length == 0:
                self.append_epoch_board(epoch_distances, self.eps, losses,
                                        "train", episode)
                self.validation_epoch(episode)
                self.dqn.save_model(name="latest_dqn.pt", forced=True)
                self.dqn.scheduler.step()
                epoch_distances = []
            episode += 1

    def init_memory(self):
        self.logger.log("Initialising memory buffer...")
        pbar = tqdm(desc="Memory buffer", total=self.init_memory_size)
        while len(self.buffer) < self.init_memory_size:
            # Reset the environment for the start of the episode.
            obs = self.env.reset()
            terminal = [False for _ in range(self.agents)]
            steps = 0
            for _ in range(self.steps_per_episode):
                steps += 1
                acts, q_values = self.get_next_actions(obs)
                obs, reward, terminal, info = self.env.step(
                    acts, q_values, terminal)
                self.buffer.append((obs, acts, reward, terminal))
                if all(t for t in terminal):
                    break
            pbar.update(steps)
        pbar.close()
        self.logger.log("Memory buffer filled")

    def validation_epoch(self, episode):
        if self.eval_env is None:
            return
        self.dqn.q_network.train(False)
        epoch_distances = []
        for k in range(self.eval_env.files.num_files):
            self.logger.log(f"eval episode {k}")
            (score, start_dists, q_values,
             info) = self.evaluator.play_one_episode()
            epoch_distances.append(
                [info['distError_' + str(i)] for i in range(self.agents)])

        val_dists = self.append_epoch_board(epoch_distances,
                                            name="eval",
                                            episode=episode)
        if (val_dists < self.best_val_distance):
            self.logger.log("Improved new best mean validation distances")
            self.best_val_distance = val_dists
            self.dqn.save_model(name="best_dqn.pt", forced=True)
        self.dqn.q_network.train(True)

    def append_episode_board(self, info, score, name="train", episode=0):
        dists = {
            str(i): info['distError_' + str(i)]
            for i in range(self.agents)
        }
        self.logger.write_to_board(f"{name}/dist", dists, episode)
        scores = {str(i): score[i] for i in range(self.agents)}
        self.logger.write_to_board(f"{name}/score", scores, episode)

    def append_epoch_board(self,
                           epoch_dists,
                           eps=0,
                           losses=[],
                           name="train",
                           episode=0):
        epoch_dists = np.array(epoch_dists)
        if name == "train":
            self.logger.write_to_board(name, {"eps": eps}, episode)
            if len(losses) > 0:
                loss_dict = {"loss": sum(losses) / len(losses)}
                self.logger.write_to_board(name, loss_dict, episode)
        for i in range(self.agents):
            mean_dist = sum(epoch_dists[:, i]) / len(epoch_dists[:, i])
            mean_dist_dict = {str(i): mean_dist}
            self.logger.write_to_board(f"{name}/mean_dist", mean_dist_dict,
                                       episode)
            min_dist_dict = {str(i): min(epoch_dists[:, i])}
            self.logger.write_to_board(f"{name}/min_dist", min_dist_dict,
                                       episode)
            max_dist_dict = {str(i): max(epoch_dists[:, i])}
            self.logger.write_to_board(f"{name}/max_dist", max_dist_dict,
                                       episode)
        return np.array(list(mean_dist_dict.values())).mean()

    def get_next_actions(self, obs_stack):
        # epsilon-greedy policy
        if np.random.random() < self.eps:
            q_values = np.zeros((self.agents, self.number_actions))
            actions = np.random.randint(self.number_actions, size=self.agents)
        else:
            actions, q_values = self.get_greedy_actions(obs_stack,
                                                        doubleLearning=True)
        return actions, q_values

    def get_greedy_actions(self, obs_stack, doubleLearning=True):
        inputs = torch.tensor(obs_stack).unsqueeze(0)
        if doubleLearning:
            q_vals = self.dqn.q_network.forward(inputs).detach().squeeze(0)
        else:
            q_vals = self.dqn.target_network.forward(inputs).detach().squeeze(
                0)
        idx = torch.max(q_vals, -1)[1]
        greedy_steps = np.array(idx, dtype=np.int32).flatten()
        return greedy_steps, q_vals.data.numpy()

    def set_reproducible(self):
        torch.manual_seed(0)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        np.random.seed(0)
Ejemplo n.º 58
0
 def test_give_a_note_on_6_random1(self):
     cube = Cube('6_random1.txt')
     evaluator = Evaluator(6)
     self.assertEqual(evaluator.give_a_note(cube), (12, 35, 84))
Ejemplo n.º 59
0
#! python 3
import csv

from evaluator import Evaluator
from my_parser.parser import Parser

text = open('botify_rules.txt', encoding='utf-8').read()
p = Parser(text)
ast = p.parse()
ev = Evaluator(ast)

with open('urls_croisierenet.csv', encoding='utf-8') as f:
    f_csv = csv.reader(f)
    for line in f_csv:
        ev.evaluate_url(line[0])
Ejemplo n.º 60
0
               frame_history=FRAME_HISTORY,
               logger=logger,
               type=args.model_name,
               collective_rewards=args.team_reward)
     model = dqn.q_network
     model.load_state_dict(torch.load(args.load, map_location=model.device))
     environment = get_player(files_list=args.files,
                              file_type=args.file_type,
                              landmark_ids=args.landmarks,
                              saveGif=args.saveGif,
                              saveVideo=args.saveVideo,
                              task=args.task,
                              agents=agents,
                              viz=args.viz,
                              logger=logger)
     evaluator = Evaluator(environment, model, logger, agents,
                           args.steps_per_episode)
     evaluator.play_n_episodes()
 else:  # train model
     environment = get_player(task='train',
                              files_list=args.files,
                              file_type=args.file_type,
                              landmark_ids=args.landmarks,
                              agents=agents,
                              viz=args.viz,
                              multiscale=args.multiscale,
                              logger=logger)
     eval_env = None
     if args.val_files is not None:
         eval_env = get_player(task='eval',
                               files_list=args.val_files,
                               file_type=args.file_type,