def __init__(self, c): self.c = c self.nnet = nnet.NNet(c.num_kernels, c.num_res_blocks) self.nnet.cuda() self.evaluator = evaluateAI.Evaluator('random', c.num_eval_games, c.max_moves) self.buffer = Buffer(c) if c.load: self.rm = jsonManager.load(c.run_manager_file_name) self.rm.c = c nnet_file_name = self.rm.number_file_name(c.nnet_file_name) # We load a nnet only if there exists one (ie. with a non zero number). if self.rm.get_number_file_name(nnet_file_name) > 0: self.nnet.load_state_dict(torch.load(nnet_file_name)) self.buffer.load() self.buffer.c = c else: self.rm = runManager.RunManager(c) # Evaluation of the initial AI. if self.rm.iter_count == 0: self.evaluation()
def run(): print "Loading data..." # load training data trainImages,trainLabels=dl.load_mnist_train() imDim = trainImages.shape[0] inputDim = 50 outputDim = 10 layerSizes = [16]*2 trainImages = trainImages.reshape(imDim**2,-1) pcer = pc.Preprocess() pcer.computePCA(trainImages) whitenedTrain = pcer.whiten(trainImages, inputDim) minibatch = whitenedTrain.shape[1] print "minibatch size: %d" % (minibatch) epochs = 10000 stepSize = 1e-2 nn = nnet.NNet(inputDim,outputDim,layerSizes,minibatch) nn.initParams() SGD = sgd.SGD(nn,alpha=stepSize,minibatch=minibatch) for e in range(epochs): print "Running epoch %d"%e SGD.run(whitenedTrain,trainLabels) SGD.dumptrace()
def run(): print "Loading data..." # load training data trainImages,trainLabels=data_loader.load_mnist_train() imDim = trainImages.shape[0] inputDim = imDim**2 outputDim = 10 layerSizes = [4048]*6 trainImages = trainImages.reshape(inputDim,-1) mbSize = 256 nn = nnet.NNet(inputDim,outputDim,layerSizes,mbSize) nn.initParams() epochs = 2 SGD = sgd.SGD(nn,epochs=20,alpha=1e-2,minibatch=mbSize,gpu=True) # run SGD loop print "Training..." SGD.run(trainImages,trainLabels)
def random(n_units, mean=None): h_network = nnet.random_linear_then_tanh_chain(n_units[:-1]) mean_network = nnet.Linear.random(n_units[-2], n_units[-1]) if mean is not None: mean_network.b.set_value(mean.astype(floatX)) sigma_network = nnet.NNet().add_layer( nnet.Linear.random(n_units[-2], n_units[-1])).add_layer(nnet.Exponential()) return GaussianSampler(h_network, mean_network, sigma_network)
def main(): config.init_config() print('go to model') print '*' * 80 _log_file = open( config.LOG_FILE_PRE + time.strftime("_%Y_%m_%d_%H%M%S", time.localtime()), 'w') # log configuration. config.log_config(_log_file) # initialize model weights_path = None if config.MODE == 2: if config.DATASET == 'WSJ0': weights_path = './_tmp_weights/ASAM_WSJ0_weight_00031.h5' elif config.DATASET == 'THCHS-30': weights_path = './_tmp_weights/ASAM_THCHS30_weight_00034.h5' dl4ss_model = nnet.NNet(_log_file, weights_path) if config.MODE == 1: print 'Start to train model ...' _log_file.write('Start to train model ...\n') dl4ss_model.train() print 'valid spk number: 2' _log_file.write('valid spk number: 2\n') dl4ss_model.predict(config.VALID_LIST, spk_num=2) print 'test spk number: 2' _log_file.write('test spk number: 2\n') dl4ss_model.predict(config.TEST_LIST, spk_num=2) print 'test spk number: 3' _log_file.write('test spk number: 3\n') dl4ss_model.predict(config.TEST_LIST, spk_num=3) print 'test spk number: 2 with bg noise' _log_file.write('test spk number: 2 with bg noise\n') dl4ss_model.predict(config.TEST_LIST, spk_num=2, add_bgd_noise=True) for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]: print 'unk spk and supplemental wav span: %02d' % supp_time _log_file.write('unk spk and supplemental wav span: %02d\n' % supp_time) dl4ss_model.predict(config.UNK_LIST, spk_num=2, unk_spk=True, supp_time=supp_time) else: print 'Wrong mode: %s' % config.MODE _log_file.write('Wrong mode: %s\n' % config.MODE) _log_file.close()
def main(): # ############## Test,测试区 ############ # ############## Test,测试区 ############ config.init_config() print('go to model') print '*' * 80 _log_file = open( config.LOG_FILE_PRE + time.strftime("_%Y_%m_%d_%H%M%S", time.localtime()), 'w') # 记录参数 config.log_config(_log_file) # 初始化网络 initialize model weights_path = './_tmp_weights/WSJ0_weight_00021.h5' weights_path = './_tmp_weights/WSJ0_weight_00150_forImages.h5' # weights_path = None dl4ss_model = nnet.NNet(_log_file, weights_path) if config.MODE == 1: print 'still train' dl4ss_model.train() elif (config.MODE == 2) and (weights_path is not None): # 在训练好的模型上测试TEST # print 'valid spk number: 2' # _log_file.write('valid spk number: 2\n') # dl4ss_model.predict(config.VALID_LIST, spk_num=2) print 'test spk number: 2' _log_file.write('test spk number: 2\n') dl4ss_model.predict(config.TEST_LIST, spk_num=2) # print 'tes spk number: 3' # _log_file.write('test spk number: 3\n') # dl4ss_model.predict(config.TEST_LIST, spk_num=3) print 'test spk number: 2 with bg noise' _log_file.write('test spk number: 2 with bg noise\n') dl4ss_model.predict(config.TEST_LIST, spk_num=2, add_bgd_noise=True) # 在训练好的模型上测试UNK # for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]: # for supp_time in [0.25, 0.5, 1, 2, 4, 8, 16, 32]: # print 'unk spk and supplemental wav span: %02d' % supp_time # _log_file.write('unk spk and supplemental wav span: %02d\n' % supp_time) # dl4ss_model.predict(config.UNK_LIST, spk_num=2, unk_spk=True, supp_time=supp_time) # else: # print 'Wrong mode: %s' % config.MODE # _log_file.write('Wrong mode: %s\n' % config.MODE) _log_file.close()
def test(opts): import editDistance as ed print "Testing model %s" % opts.inFile phone_map = get_phone_map_swbd() with open(opts.inFile, 'r') as fid: old_opts = pickle.load(fid) _ = pickle.load(fid) _ = pickle.load(fid) loader = dl.DataLoader(opts.dataDir, old_opts.rawDim, old_opts.inputDim) if 'layers' not in dir(old_opts): old_opts.layers = [old_opts.layerSize] * old_opts.numLayers nn = nnet.NNet(old_opts.inputDim, old_opts.outputDim, old_opts.layers, train=False) nn.initParams() nn.fromFile(fid) totdist = numphones = 0 fid = open('hyp.txt', 'w') for i in range(1, opts.numFiles + 1): data_dict, alis, keys, sizes = loader.loadDataFileDict(i) for k in keys: gp.free_reuse_cache() hyp = nn.costAndGrad(data_dict[k]) hyp = [phone_map[h] for h in hyp] ref = [phone_map[int(r)] for r in alis[k]] dist, ins, dels, subs, corr = ed.edit_distance(ref, hyp) print "Distance %d/%d" % (dist, len(ref)) fid.write(k + ' ' + ' '.join(hyp) + '\n') totdist += dist numphones += len(alis[k]) fid.close() print "PER : %f" % (100 * totdist / float(numphones))
def __init__(self, horizon=1): super(NeuralNetwork, self).__init__(window_size=1, horizon=horizon) import nnet self.nnet_instance = nnet.NNet() self.nnet_instance.compile_model()
def __init__(self, param): self.param = param def costAndGradVec(self, param): return np.sum(np.array(param)**2), 2 * np.array(param) def paramVec(self): return self.param if __name__ == '__main__': # gcc = gradcheckCheck(np.random.rand(10,1)) # gradcheck(gcc) inputDim = 5 outputDim = 10 layerSizes = [3, 3] mbSize = 5 # fake data data = gp.rand(inputDim, mbSize) import random labels = [random.randint(0, 9)] * mbSize # make nnet nn = nnet.NNet(inputDim, outputDim, layerSizes, mbSize, train=True) nn.initParams() # run gradcheck(nn, 1000, data, labels)
def load_nnet(self): self.nnet = nnet.NNet(self.c.num_kernels, self.c.num_res_blocks) self.nnet.cuda() self.nnet.load_state_dict(torch.load(self.c.nnet_file_name))
def main(): pa = parameters.Parameters() type_exp = 'pg_re' # 'pg_su' 'pg_su_compact' 'v_su', 'pg_v_re', 'pg_re', q_re', 'test' pg_resume = None v_resume = None q_resume = None log = None render = False plot = False try: opts, args = getopt.getopt(sys.argv[1:], "hi:o:", [ "exp_type=", "num_res=", "num_nw=", "simu_len=", "num_ex=", "num_seq_per_batch=", "eps_max_len=", "num_epochs=", "time_horizon=", "res_slot=", "max_job_len=", "max_job_size=", "new_job_rate=", "dist=", "lr_rate=", "ba_size=", "pg_re=", "v_re=", "q_re=", "out_freq=", "ofile=", "log=", "render=", "unseen=", "plot=" ]) except getopt.GetoptError: script_usage() sys.exit(2) for opt, arg in opts: if opt == '-h': script_usage() sys.exit() elif opt in ("-e", "--exp_type"): type_exp = arg elif opt in ("-n", "--num_res"): pa.num_resources = int(arg) elif opt in ("-w", "--num_nw"): pa.num_nw = int(arg) elif opt in ("-s", "--simu_len"): pa.simu_len = int(arg) elif opt in ("-n", "--num_ex"): pa.num_ex = int(arg) elif opt in ("-sp", "--num_seq_per_batch"): pa.num_seq_per_batch = int(arg) elif opt in ("-el", "--eps_max_len"): pa.episode_max_length = int(arg) elif opt in ("-ne", "--num_epochs"): pa.num_epochs = int(arg) elif opt in ("-t", "--time_horizon"): pa.time_horizon = int(arg) elif opt in ("-rs", "--res_slot"): pa.res_slot = int(arg) elif opt in ("-ml", "--max_job_len"): pa.max_job_len = int(arg) elif opt in ("-ms", "--max_job_size"): pa.max_job_size = int(arg) elif opt in ("-nr", "--new_job_rate"): pa.new_job_rate = float(arg) elif opt in ("-d", "--dist"): pa.discount = float(arg) elif opt in ("-l", "--lr_rate"): pa.lr_rate = float(arg) elif opt in ("-b", "--ba_size"): pa.batch_size = int(arg) elif opt in ("-p", "--pg_re"): pg_resume = arg elif opt in ("-v", "--v_re"): v_resume = arg elif opt in ("-q", "--q_re"): q_resume = arg elif opt in ("-f", "--out_freq"): pa.output_freq = int(arg) elif opt in ("-o", "--ofile"): pa.output_filename = arg elif opt in ("-lg", "--log"): log = arg elif opt in ("-r", "--render"): render = (arg == 'True') elif opt in ("-pl", "--plot"): plot = (arg == 'True') elif opt in ("-u", "--unseen"): pa.generate_unseen = (arg == 'True') else: script_usage() sys.exit() if log is not None: orig_stdout = sys.stdout f = open(log, 'w') sys.stdout = f if pg_resume is None: print("PG resume is empty!") sys.exit(1) pa.compute_dependent_parameters() repre = 'image' end = 'all_done' env = environment.Env(pa, render=render, repre=repre, end=end) pg_learner = pg_network.PGLearner(pa) net_handle = open(pg_resume, 'rb') net_params = cPickle.load(net_handle) pg_learner.set_net_params(net_params) outputFileName = pa.output_filename + '_' + \ ntpath.basename(pg_resume) + '_test.pkl' pg_learner.write_net_to_nnet(outputFileName) nnetFilename = outputFileName + '.nnet' r = nnet.NNet(nnetFilename) smallHigherBound = 1.0e-10 smallLowerBound = -1.0e-10 for wIdx, w in enumerate(r.weights): smallRows = np.all((w <= smallHigherBound) & (w >= smallLowerBound), axis=1) smallRowsIndices = np.where(smallRows == True) for row in smallRowsIndices[0]: rowBias = r.biases[wIdx][row] #add it's bias to all biasses of next layer since we assume fully connected if ((wIdx + 1) < len(r.biases)): r.biases[wIdx + 1] = r.biases[wIdx + 1] + rowBias #Now we delete the lines with 'dead' neurons r.weights[wIdx] = np.delete(w, smallRowsIndices[0], axis=0) if ((wIdx + 1) < len(r.weights)): r.weights[wIdx + 1] = np.delete(r.weights[wIdx + 1], smallRowsIndices[0], axis=1) r.biases[wIdx] = np.delete(r.biases[wIdx], smallRowsIndices[0], axis=0) # now we export the file once again after the 'dead' neuron filtration for wIdx, w in enumerate(r.weights): r.weights[wIdx] = r.weights[wIdx].transpose() writeNNet.writeNNet(r.weights, r.biases, r.mins, r.maxes, r.means, r.ranges, outputFileName + '_cleaned_' + '.nnet') if log is not None: sys.stdout = orig_stdout f.close()
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage=usage) parser.add_option("--test", action="store_true", dest="test", default=False) # Architecture parser.add_option("--layers", dest="layers", type="string", default="100,100", help="layer1size,layer2size,...,layernsize") parser.add_option("--temporal_layer", dest="temporalLayer", type="int", default=-1) # Optimization parser.add_option("--optimizer", dest="optimizer", type="string", default="momentum") parser.add_option("--momentum", dest="momentum", type="float", default=0.9) parser.add_option("--epochs", dest="epochs", type="int", default=1) parser.add_option("--step", dest="step", type="float", default=1e-4) parser.add_option( "--anneal", dest="anneal", type="float", default=1, help="Sets (learning rate := learning rate / anneal) after each epoch." ) # Data parser.add_option( "--dataDir", dest="dataDir", type="string", default= "/scail/group/deeplearning/speech/awni/kaldi-stanford/kaldi-trunk/egs/swbd/s5b/exp/train_ctc/" ) parser.add_option("--numFiles", dest="numFiles", type="int", default=384) parser.add_option("--inputDim", dest="inputDim", type="int", default=41 * 15) parser.add_option("--rawDim", dest="rawDim", type="int", default=41 * 15) parser.add_option("--outputDim", dest="outputDim", type="int", default=34) parser.add_option("--outFile", dest="outFile", type="string", default="models/test.bin") parser.add_option("--inFile", dest="inFile", type="string", default=None) (opts, args) = parser.parse_args(args) opts.layers = [int(l) for l in opts.layers.split(',')] # Testing if opts.test: test(opts) return loader = dl.DataLoader(opts.dataDir, opts.rawDim, opts.inputDim) #NOTE at some point we need to unify the nnet and rnnet modules nn = None if opts.temporalLayer > 0: nn = rnnet.RNNet(opts.inputDim, opts.outputDim, opts.layers, opts.temporalLayer) else: nn = nnet.NNet(opts.inputDim, opts.outputDim, opts.layers) nn.initParams() # Load model if exists if opts.inFile is not None: with open(opts.inFile, 'r') as fid: _ = pickle.load(fid) _ = pickle.load(fid) _ = pickle.load(fid) nn.fromFile(fid) SGD = sgd.SGD(nn, alpha=opts.step, optimizer=opts.optimizer, momentum=opts.momentum) # Setup some random keys for tracing with open('randKeys.bin', 'r') as fid: traceK = pickle.load(fid) for k in traceK: nn.hist[k] = [] # write initial model to disk with open(opts.outFile, 'w') as fid: pickle.dump(opts, fid) pickle.dump(SGD.costt, fid) pickle.dump(nn.hist, fid) nn.toFile(fid) # Training import time for _ in range(opts.epochs): for i in np.random.permutation(opts.numFiles) + 1: start = time.time() data_dict, alis, keys, sizes = loader.loadDataFileDict(i) SGD.run_seq(data_dict, alis, keys, sizes) end = time.time() print "File time %f" % (end - start) # Save anneal after epoch SGD.alpha = SGD.alpha / opts.anneal with open(opts.outFile, 'w') as fid: pickle.dump(opts, fid) pickle.dump(SGD.costt, fid) pickle.dump(nn.hist, fid) nn.toFile(fid)
if __name__ == '__main__': # gcc = gradcheckCheck(np.random.rand(10,1)) # gradcheck(gcc) inputDim = 5 numPhones = 6 outputDim = numPhones + 1 seq_len_out = 4 seq_len_in = 10 # can't output more symbols than input times assert seq_len_in > seq_len_out layerSizes = [10, 10] # make sure seq labels do not have '0' which is our blank index label_seq = np.floor(np.random.rand(seq_len_out, 1) * numPhones) label_seq = 1 + label_seq.astype(np.int32) data = np.random.randn(inputDim, seq_len_in) # make nnet nn = nnet.NNet(inputDim, outputDim, layerSizes, seq_len_in, train=True) nn.initParams() # run cost, grad = nn.costAndGrad(data, label_seq) print cost # run gradcheck(nn, 1000, data, label_seq)
import numpy as np import mnist import nnet if __name__ == '__main__': trX, teX, trY, teY = mnist.mnist() net = nnet.NNet([50], l1_reg=0.0002, dropout=0.4) net.train(trX, trY, epochs=10) wrong = 0 total = 0 for X, y in zip(teX, teY): prediction = net.predict(np.array([X])) actual = np.argmax(y) if prediction != actual: wrong += 1 total += 1 print(wrong, total, float(total - wrong) / float(total))