def nn_back(name): instances = read_dataset('./spambase.csv', [1]) train_set, test_set = train_test_split(instances, 0.3) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], HyperbolicTangentSigmoid()) bp_instance = BatchBackPropagationTrainer(train_set, classification_network, measure, RPROPUpdateRule()) nn_state = {'network': classification_network, 'trainer': bp_instance} wrapper_nn = AlgoWrapper( nn_state, lambda state: state['trainer'].train(), lambda state: classification_error_acc(train_set, state[ 'network'], measure), lambda state: classification_error_acc( test_set, state['network'], measure)) # create name and invalidate if super empty decorated_name = name timed_trainer = TimedTrainer(decorated_name, wrapper_nn, 200, 4000, 1, _param_dict={'name': name}) timed_trainer.run() print "NNBP bp done"
def main(): training_ints = initialize_instances('data/bank_train.csv') testing_ints = initialize_instances('data/bank_test.csv') factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) acti = LogisticSigmoid() rule = RPROPUpdateRule() ######################### back prop ##################### classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1, OUTPUT_LAYER],acti) train(BatchBackPropagationTrainer(data_set,classification_network,measure,rule,), classification_network, 'Backprop', training_ints,testing_ints, measure, './ANN/BP/BACKPROP_LOG.csv', 2000) ######################### simulated annealing ################# for CE in [0.15,0.35,0.55,0.75,0.95]: for T in [1e8,1e10,1e12]: classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1, OUTPUT_LAYER],acti) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oFile = "./ANN/SA/%s_%s_LOG.csv"%(CE,T) train(SimulatedAnnealing(T, CE, nnop), classification_network, 'simulated annealing', training_ints, testing_ints, measure, oFile, 2000) ######################### random hill climbing ################# classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1, OUTPUT_LAYER],acti) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) train(RandomizedHillClimbing(nnop), classification_network, 'RHC', training_ints, testing_ints, measure, './ANN/RHC/RHC_LOG.csv', 2000) ######################### genetic algorithm ################# for P in [100]: for mate in [5, 15, 30]: for mutate in [5,15,30]: classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1, OUTPUT_LAYER],acti) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oFile = "./ANN/GA/%s_%s_%s_LOG.csv"%(P, mate, mutate) train(StandardGeneticAlgorithm(P, mate, mutate, nnop), classification_network, 'GA', training_ints, testing_ints, measure, oFile, 2000)
def main(): """Run this experiment""" training_ints = initialize_instances('titanic_train.csv') testing_ints = initialize_instances('titanic_test.csv') factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) acti = LogisticSigmoid() rule = RPROPUpdateRule() oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1, OUTPUT_LAYER],acti) train(BatchBackPropagationTrainer(data_set,classification_network,measure,rule), classification_network, 'Backprop', training_ints,testing_ints, measure)
def main(): """Run algorithms on the adult dataset.""" train_instances = initialize_instances(trainX) test_instances = initialize_instances(testX) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(train_instances) networks = [] # BackPropagationNetwork nnop = [] # NeuralNetworkOptimizationProblem oa = [] # OptimizationAlgorithm oa_names = ["RHC", "SA", "GA", "BP"] print(sys.argv) if len(sys.argv) > 1: oa_names = [sys.argv[1]] set_num = sys.argv[2] # results = "" for name in oa_names: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], RELU()) networks.append(classification_network) if name != "BP": nnop.append( NeuralNetworkOptimizationProblem(data_set, classification_network, measure)) else: print("adding backprop") rule = RPROPUpdateRule() nnop.append( BatchBackPropagationTrainer(data_set, classification_network, measure, rule)) if "RHC" in oa_names: rhc_index = oa_names.index("RHC") oa.append(RandomizedHillClimbing(nnop[rhc_index])) if "SA" in oa_names: sa_index = oa_names.index("SA") oa.append(SimulatedAnnealing(1E11, .95, nnop[sa_index])) if "GA" in oa_names: ga_index = oa_names.index("GA") oa.append(StandardGeneticAlgorithm(100, 50, 10, nnop[ga_index])) if "BP" in oa_names: rule = RPROPUpdateRule() bp_index = oa_names.index("BP") oa.append(nnop[bp_index]) for i, name in enumerate(oa_names): train(oa[i], networks[i], oa_names[i], train_instances, test_instances, measure)
def main(): train_data = initialize_instances(TRAIN_FILE) test_data = initialize_instances(TEST_FILE) # Get data factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(train_data) activation = RELU() rule = RPROPUpdateRule() oa_name = 'Backprop' classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER], activation) oa = BatchBackPropagationTrainer(data_set, classification_network, measure, rule) train(oa, classification_network, oa_name, train_data, test_data, measure)
def main(): """Run this experiment""" training_ints = initialize_instances('SeasonsStats_trg.csv') testing_ints = initialize_instances('SeasonsStats_test.csv') validation_ints = initialize_instances('SeasonsStats_val.csv') for t in range(TRIALS): factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) relu = RELU() rule = RPROPUpdateRule() oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork([INPUT_LAYER, HIDDEN_LAYER1,HIDDEN_LAYER2,HIDDEN_LAYER3, OUTPUT_LAYER],relu) train(BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network,'Backprop', training_ints, validation_ints, testing_ints, measure)
def run_experiment(self, train, test, validation): """Run experiment Args: train (list): List of training instances. test (list): List of test instances. validation (list): List of validation instances. """ factory = BackPropagationNetworkFactory() # instantiate main NN class params = [ self.input_layer, self.hidden_layer_one, self.hidden_layer_two, self.output_layer ] self.network = factory.createClassificationNetwork(params) dataset = DataSet(train) # setup training instances dataset nnop = NeuralNetworkOptimizationProblem(dataset, self.network, self.measure) oa = None # get output file name outpath = 'results/NN' filename = None # options for different optimization algorithms if self.oaName == 'BP': filename = '{}/results.csv'.format(self.oaName) rule = RPROPUpdateRule() oa = BatchBackPropagationTrainer(dataset, self.network, self.measure, rule) elif self.oaName == 'RHC': filename = '{}/results.csv'.format(self.oaName) oa = RandomizedHillClimbing(nnop) elif self.oaName == 'SA': filename = '{}/results_{}_{}.csv'.format(self.oaName, self.SA_T, self.SA_C) oa = SimulatedAnnealing(self.SA_T, self.SA_C, nnop) elif self.oaName == 'GA': filename = '{}/results_{}_{}_{}.csv'.format( self.oaName, self.GA_P, self.GA_MA, self.GA_MU) oa = StandardGeneticAlgorithm(self.GA_P, self.GA_MA, self.GA_MU, nnop) # train network filepath = get_abspath(filename, outpath) self.train(oa, train, test, validation, filepath)
def main(): """Run this experiment""" training_ints = initialize_instances(TRAIN_DATA_FILE) testing_ints = initialize_instances(TEST_DATA_FILE) validation_ints = initialize_instances(VALIDATE_DATA_FILE) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) relu = RELU() # 50 and 0.000001 are the defaults from RPROPUpdateRule.java rule = RPROPUpdateRule(0.064, 50, 0.000001) oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER], relu) train(BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', training_ints, validation_ints, testing_ints, measure, TRAINING_ITERATIONS, OUTFILE.format('Backprop'))
def main(): """Run this experiment""" training_ints = initialize_instances('./../data/x_train_val.csv') testing_ints = initialize_instances('./../data/x_test.csv') factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) acti = HyperbolicTangentSigmoid() rule = RPROPUpdateRule() # oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork([ INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, HIDDEN_LAYER3, HIDDEN_LAYER4, OUTPUT_LAYER ], acti) train( BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', training_ints, testing_ints, measure, TRAINING_ITERATIONS, OUTFILE)
def main(layers, training_iterations, test_data_file, train_data_file, validate_data_file, data_name): """Run this experiment""" training_ints = base.initialize_instances(train_data_file) testing_ints = base.initialize_instances(test_data_file) validation_ints = base.initialize_instances(validate_data_file) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) relu = RELU() # 50 and 0.000001 are the defaults from RPROPUpdateRule.java rule = RPROPUpdateRule(0.064, 50, 0.000001) oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork(layers, relu) base.train( BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', training_ints, validation_ints, testing_ints, measure, training_iterations, OUTFILE.format(data_name, 'Backprop')) return
def main(ds_name): """Run this experiment""" nn_config, train_file, val_file, test_file = get_problemset(ds_name) training_ints = initialize_instances(train_file) testing_ints = initialize_instances(test_file) validation_ints = initialize_instances(val_file) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) relu = RELU() # 50 and 0.000001 are the defaults from RPROPUpdateRule.java rule = RPROPUpdateRule(0.064, 50, 0.000001) oa_names = "Backprop_{}".format(name) classification_network = factory.createClassificationNetwork( nn_config, relu) train( BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, oa_names, training_ints, validation_ints, testing_ints, measure, TRAINING_ITERATIONS, OUTFILE.format(oa_names))
def Backpropogation(out_path, train_inst, test_inst, repeats, training_iterations): for i in range(repeats): out_path_ = out_path.replace("BP_", 'BP_{}'.format(str(i).zfill(3))) with open(out_path_, 'w') as f: f.write('{},{},{},{},{},{}\n'.format('iteration', 'MSE_trg', 'MSE_tst', 'acc_trg', 'acc_tst', 'elapsed')) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(train_inst) # acti = LogisticSigmoid() acti = HyperbolicTangentSigmoid() rule = RPROPUpdateRule() classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER], acti) train( BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', train_inst, test_inst, measure, training_iterations, out_path_)
def main(): """Run this experiment""" training_data = initialize_instances('../data/Pima-train.csv') testing_data = initialize_instances('../data/Pima-test.csv') print(len(training_data)) #testing_ints = initialize_instances('m_test.csv') #validation_ints = initialize_instances('m_val.csv') factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_data) relu = RELU() rule = RPROPUpdateRule() oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER], relu) train( BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', training_data, testing_data, measure)
def main(layers, training_iterations, test_data_file, train_data_file, validate_data_file, data_name): """Run this experiment""" training_ints = base.initialize_instances(train_data_file) testing_ints = base.initialize_instances(test_data_file) validation_ints = base.initialize_instances(validate_data_file) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(training_ints) relu = RELU() # 50 and 0.000001 are the defaults from RPROPUpdateRule.java rule = RPROPUpdateRule(0.064, 50, 0.000001) oa_names = ["Backprop"] classification_network = factory.createClassificationNetwork(layers, relu) with open(OUTFILE.format(data_name, 'Backprop'), 'a+') as f: content = f.read() if "MSE_trg" not in content: f.write('{},{},{},{},{},{},{},{},{},{},{}\n'.format('iteration', 'MSE_trg', 'MSE_val', 'MSE_tst', 'acc_trg', 'acc_val', 'acc_tst', 'f1_trg', 'f1_val', 'f1_tst', 'elapsed')) base.train(BatchBackPropagationTrainer(data_set, classification_network, measure, rule), classification_network, 'Backprop', training_ints, validation_ints, testing_ints, measure, training_iterations, OUTFILE.format(data_name, 'Backprop')) return
def main(): """Run algorithms on the abalone dataset.""" train_instances = initialize_instances() test_instances = initialize_instances(test=True) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(train_instances) networks = [] # BackPropagationNetwork oa = [] # OptimizationAlgorithm oa_names = [] if do_rhc: oa_names.append("RHC") if do_sa: oa_names.append("SA") if do_ga: oa_names.append("GA") if do_bp: oa_names.append("BP") results = "" # For each algo, need to see if we are doing sweeps # No need to sweep rhc as there are no parameters if do_rhc and sweep == False: training_iter = TRAINING_ITERATIONS if do_fmnist: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER]) if do_chess: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER]) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oa = RandomizedHillClimbing(nnop) name = "RHC" train(oa, classification_network, name, train_instances, measure, training_iter, test_instances, True) if do_sa: training_iter = TRAINING_ITERATIONS count = 0 for temp, cooling in product(sa_temp, sa_cooling): if do_fmnist: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER]) if do_chess: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER]) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oa = SimulatedAnnealing(temp, cooling, nnop) name = "SA_sweep" if count == 0: print_head = True else: print_head = False train(oa, classification_network, name, train_instances, measure, training_iter, test_instances, print_head, temp, cooling) count += 1 if do_ga: training_iter = GA_TRAINING_ITERATIONS count = 0 for pop, prop_mate, prop_mutate in product(ga_pop, ga_prop_mate, ga_prop_mutate): if do_fmnist: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER]) if do_chess: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER]) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) mate = int(math.floor(pop * prop_mate)) mutate = int(math.floor(pop * prop_mutate)) oa = StandardGeneticAlgorithm(pop, mate, mutate, nnop) name = "GA_sweep" if count == 0: print_head = True else: print_head = False train(oa, classification_network, name, train_instances, measure, training_iter, test_instances, print_head, pop, prop_mate, prop_mutate) count += 1 if do_bp and sweep == False: training_iter = TRAINING_ITERATIONS if do_fmnist: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER]) if do_chess: classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER1, HIDDEN_LAYER2, OUTPUT_LAYER]) oa = BatchBackPropagationTrainer(data_set, classification_network, measure, RPROPUpdateRule()) name = "BP" train(oa, classification_network, name, train_instances, measure, training_iter, test_instances, True)
def main(trainfile, testfile, validfile, oa_name, i, params): print("== [{}] ==".format(oa_name)) res = {} #for i in range(25): res[i] = {} if i == 9: print("Invalid i %d" % (i)) sys.exit(1) print("LABEL: {}".format(i)) traininstances = initialize_instances(trainfile, i) testinstances = initialize_instances(testfile, i) validinstances = initialize_instances(validfile, i) factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(traininstances) rule = RPROPUpdateRule() # was networks[] classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER]) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oa = None # was oa = [] suffix = "" if oa_name == "BP": oa = BatchBackPropagationTrainer(data_set, classification_network, measure, rule) if oa_name == "RHC": oa = RandomizedHillClimbing(nnop) if oa_name == "SA": suffix = '-' + '-'.join(params) oa = SimulatedAnnealing(float(params[0]), float(params[1]), nnop) if oa_name == "GA": suffix = '-' + '-'.join(params) oa = StandardGeneticAlgorithm(int(params[0]), int(params[1]), int(params[2]), nnop) ttvinstances = { 'train': traininstances, 'test': testinstances, 'valid': validinstances } train_start = timeit.default_timer() train(oa, classification_network, oa_name, ttvinstances, measure, i, suffix) train_end = timeit.default_timer() print 'train time: %d secs' % (int(train_end - train_start)) if oa_name != "BP": optimal_instance = oa.getOptimal() classification_network.setWeights(optimal_instance.getData()) ttvinstances = { 'train': traininstances, 'valid': validinstances, 'test': testinstances } for key, instances in zip(ttvinstances.keys(), ttvinstances.values()): query_start = timeit.default_timer() tp = 0. fp = 0. fn = 0. tn = 0. precision = 0. recall = 0. f1 = 0. print "scoring %s..." % (key) for instance in instances: classification_network.setInputValues(instance.getData()) classification_network.run() actual = instance.getLabel().getContinuous() predicted = classification_network.getOutputValues().get(0) #print ('actual = %.3f, predicted = %.3f' % (actual, predicted)) if actual == 1.: if predicted >= 0.5: tp += 1. else: fn += 1. else: if predicted >= 0.5: fp += 1. else: tn += 1. query_end = timeit.default_timer() if tp + fp > 0.: precision = tp / (tp + fp) if fn + tp > 0.: recall = tp / (fn + tp) if precision + recall > 0.: f1 = 2. * precision * recall / (precision + recall) correct = tp + tn total = correct + fp + fn print "%s f1 = %0.10f" % (key, f1) print "%s accuracy = %0.10f" % (key, correct / total) print "%s query time: %d secs" % (key, int(query_end - query_start))
def run_all(): dataSource = 'wine' INPUT_LAYER = 13 HIDDEN_LAYER = 100 OUTPUT_LAYER = 1 # dataSource = 'wage' # INPUT_LAYER = 106 # HIDDEN_LAYER = 1000 # OUTPUT_LAYER = 1 train_data = initialize_instances('data/balanced_' + dataSource + '_cleaned_train.csv') test_data = initialize_instances('data/balanced_' + dataSource + '_cleaned_test.csv') factory = BackPropagationNetworkFactory() measure = SumOfSquaresError() data_set = DataSet(train_data) update_rule = RPROPUpdateRule() alg = 'backprop' classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], RELU()) oa = BatchBackPropagationTrainer(data_set, classification_network, measure, update_rule) fit = oa run(alg, oa, fit, classification_network, measure, train_data, test_data, dataSource) alg = 'RHC' classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], RELU()) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) oa = RandomizedHillClimbing(nnop) iters = 1 fit = FixedIterationTrainer(oa, iters) run(alg, oa, fit, classification_network, measure, train_data, test_data, dataSource) alg = 'SA' classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], RELU()) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) startTemp = 1E10 coolingFactor = .8 oa = SimulatedAnnealing(startTemp, coolingFactor, nnop) iters = 1 fit = FixedIterationTrainer(oa, iters) run(alg, oa, fit, classification_network, measure, train_data, test_data, dataSource) alg = 'GA' classification_network = factory.createClassificationNetwork( [INPUT_LAYER, HIDDEN_LAYER, OUTPUT_LAYER], RELU()) nnop = NeuralNetworkOptimizationProblem(data_set, classification_network, measure) population = 200 mates = 50 mutations = 10 oa = StandardGeneticAlgorithm(population, mates, mutations, nnop) iters = 1 fit = FixedIterationTrainer(oa, iters) run(alg, oa, fit, classification_network, measure, train_data, test_data, dataSource)