def train(train_loader, num_classes): parser = argparse.ArgumentParser(description="ReID Baseline Training") parser.add_argument( "--config_file", default="", help="path to config file", type=str ) parser.add_argument("opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER) args = parser.parse_args() if args.config_file != "": cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() set_seed(cfg.SOLVER.SEED) output_dir = cfg.OUTPUT_DIR if output_dir and not os.path.exists(output_dir): os.makedirs(output_dir) logger = setup_logger("reid_baseline", output_dir, if_train=True) logger.info("Saving model in the path :{}".format(cfg.OUTPUT_DIR)) logger.info(args) if args.config_file != "": logger.info("Loaded configuration file {}".format(args.config_file)) with open(args.config_file, 'r') as cf: config_str = "\n" + cf.read() logger.info(config_str) logger.info("Running with config:\n{}".format(cfg)) os.environ['CUDA_VISIBLE_DEVICES'] = cfg.MODEL.DEVICE_ID if cfg.MODEL.PRETRAIN_CHOICE == 'finetune': model = make_model(cfg, num_class=num_classes) model.load_param_finetune(cfg.MODEL.PRETRAIN_PATH) print('Loading pretrained model for finetuning......') else: model = make_model(cfg, num_class=num_classes) loss_func = make_loss(cfg, num_classes=num_classes) optimizer = make_optimizer(cfg, model) scheduler = WarmupCosineAnnealingLR(optimizer, cfg.SOLVER.MAX_EPOCHS, cfg.SOLVER.DELAY_ITERS, cfg.SOLVER.ETA_MIN_LR, cfg.SOLVER.WARMUP_FACTOR, cfg.SOLVER.WARMUP_EPOCHS, cfg.SOLVER.WARMUP_METHOD) logger.info("use WarmupCosineAnnealingLR, delay_step:{}".format(cfg.SOLVER.DELAY_ITERS)) do_train( cfg, model, train_loader, optimizer, scheduler, # modify for using self trained model loss_func )
def check_video_face(): if request.method == "POST": face_capture.perform() model.make_model() return render_template('video_kyc.html', context="Model created Successfully") else: return render_template('video_kyc.html', context='')
def test_make_model(self): model = make_model() model_3_topics = make_model(3) self.assertEqual(model.n_components, 5, "False default num of components") self.assertEqual(model_3_topics.n_components, 3, "False num of components passed to model") self.assertEqual(model.solver, "mu", "False solver in model") self.assertEqual(model.beta_loss, "kullback-leibler", "False kind of beta loss") self.assertEqual(model.alpha, 0.1, "False alpha value")
def main(disp_text=True): if config.fresh_model: config.all_losses = [] save_model(make_model()) model = load_model() if disp_text: print('created model.', end=' ') else: model = load_model() if not model: save_model(make_model()) model = load_model() if disp_text: print('created model.', end=' ') else: if disp_text: print('loaded model.', end=' ') data = load_data() data, data_dev = split_data(data) data = [d for i,d in enumerate(data) if i in [8,10,13,14]] print() seq_lens = [len(d) for d in data] print(f'seq lens: {seq_lens}') min_seq_len = min(seq_lens) print(f'min seq len: {min_seq_len}') if not config.max_seq_len or config.max_seq_len > min_seq_len: config.max_seq_len = min_seq_len data = [d[:config.max_seq_len] for d in data] # from random import choice # from torch import randn # data = [[randn(config.in_size) for _ in range(choice(range(config.max_seq_len//2,config.max_seq_len)))] for _ in range(10)] # data_dev = [] # for d in data: print(len(d)) if not config.batch_size or config.batch_size >= len(data): config.batch_size = len(data) elif config.batch_size < 1: config.batch_size = int(len(data)*config.batch_size) if disp_text: print(f'hm data: {len(data)}, hm dev: {len(data_dev)}, bs: {config.batch_size}, lr: {config.learning_rate}, \ntraining started @ {now()}') for ep in range(config.hm_epochs): for i, batch in enumerate(batchify_data(data)): train_on(model, batch) return model
def train(params): batch_size = 64 train_dataloader = torch.utils.data.DataLoader(ListDataset(['data/dataset/train.txt']), batch_size=batch_size, shuffle=True, num_workers=0) val_dataloader = torch.utils.data.DataLoader(ListDataset('data/dataset/validation.txt'), batch_size=batch_size, shuffle=False, num_workers=0) model = make_model(len(char2token)) model.to(device=params['device']) criterion = LabelSmoothing(size=len(char2token), padding_idx=0, smoothing=0.1) criterion.to(device=params['device']) model_opt = NoamOpt(model.tgt_embed[0].d_model, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(10000): params['epoch'] = epoch model.train() train_loss = run_epoch(train_dataloader, model, SimpleLossCompute(model.generator, criterion, model_opt), params) print("train_loss", train_loss) model.eval() val_loss = run_epoch(val_dataloader, model, SimpleLossCompute(model.generator, criterion, None), params) print("val_loss", val_loss) torch.save(model.state_dict(), 'checkpoint/%08d_%f.pth' % (epoch, val_loss))
def slave(): #packet = (np.empty(SOLUTION_PACKET_SIZE, dtype=np.int32),'') while 1: packet = comm.recv(source=0) current_env_name = packet['current_env_name'] packet = packet['result'] #packet, current_env_name = packet assert (len(packet) == SOLUTION_PACKET_SIZE) solutions = decode_solution_packet(packet) results = [] new_model = make_model() new_model.make_env(current_env_name) for solution in solutions: worker_id, jobidx, seed, train_mode, max_len, weights = solution assert (train_mode == 1 or train_mode == 0), str(train_mode) worker_id = int(worker_id) possible_error = "work_id = " + str(worker_id) + " rank = " + str( rank) assert worker_id == rank, possible_error jobidx = int(jobidx) seed = int(seed) fitness, timesteps = worker(weights, seed, max_len, new_model, train_mode) results.append([worker_id, jobidx, fitness, timesteps]) result_packet = encode_result_packet(results) assert len(result_packet) == RESULT_PACKET_SIZE comm.Send(result_packet, dest=0)
def client(): new_model = make_model(sys.argv[1]) while True: packet = comm.recv(source=0) packet = packet['result'] solutions = decode_solution_packet(packet) results = [] new_model.make_env() for solution in solutions: worker_id, jobidx, seed, train_mode, max_len, weights = solution worker_id = int(worker_id) jobidx = int(jobidx) seed = int(seed) fitness, timesteps = worker(weights, seed, max_len, new_model, train_mode) results.append([worker_id, jobidx, fitness, timesteps]) new_model.env.close() result_packet = encode_result_packet(results) comm.Send(result_packet, dest=0)
def init_model(): from setting import LAYERS, D_MODEL, D_FF, DROPOUT, H_NUM, TGT_VOCAB, SRC_VOCAB from model import make_model # 模型的初始化 model = make_model(SRC_VOCAB, TGT_VOCAB, LAYERS, D_MODEL, D_FF, H_NUM, DROPOUT) return model
def initialize_settings(sigma_init=0.1, sigma_decay=0.9999): global population, filebase, game, model, num_params, es, PRECISION, SOLUTION_PACKET_SIZE, RESULT_PACKET_SIZE population = num_worker * num_worker_trial filebase = 'log/' + gamename + '.' + optimizer + '.' + str( num_episode) + '.' + str(population) game = config.games[gamename] model = make_model(game) num_params = model.param_count print("size of model", num_params) if optimizer == 'ses': ses = PEPG(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_alpha=0.2, sigma_limit=0.02, elite_ratio=0.1, weight_decay=0.005, popsize=population) es = ses elif optimizer == 'ga': ga = SimpleGA(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_limit=0.02, elite_ratio=0.1, weight_decay=0.005, popsize=population) es = ga elif optimizer == 'cma': cma = CMAES(num_params, sigma_init=sigma_init, popsize=population) es = cma elif optimizer == 'pepg': pepg = PEPG(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_alpha=0.20, sigma_limit=0.02, learning_rate=0.01, learning_rate_decay=1.0, learning_rate_limit=0.01, weight_decay=0.005, popsize=population) es = pepg else: oes = OpenES(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_limit=0.02, learning_rate=0.01, learning_rate_decay=1.0, learning_rate_limit=0.01, antithetic=antithetic, weight_decay=0.005, popsize=population) es = oes PRECISION = 10000 SOLUTION_PACKET_SIZE = (5 + num_params) * num_worker_trial RESULT_PACKET_SIZE = 4 * num_worker_trial
def predict(load_path): model = make_model() model.load_weights(load_path + "weights_best.hdf5") sgd = SGD(lr=0.002, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss="categorical_crossentropy", metrics=['accuracy']) for fold in clips: print(fold) for clip in clips[fold]: print(clip) x = [] x.append(get_features(clip, 0)) x = np.array(x) clip_prediction = model.predict(x, 1) for i in range(1, clip.PElog_spectra.shape[2]): x = [] x.append(get_features(clip, i)) x = np.array(x) clip_prediction *= model.predict( x, 1) # multiply probabilities for the segments print(np.argmax(prediction), clip.target) print(clip_prediction[0][int(clip.target)])
def __init__(self, env, num_episodes, expert_file): self.env = env self.expert = tf.keras.models.load_model(expert_file) self.num_episodes = num_episodes print("Expert insput shape ", self.expert.input_shape) self.model = make_model() print("Model input shape ", self.model.input_shape)
def train(): batch_size = 64 train_dataloader = torch.utils.data.DataLoader(ListDataset( ['your-train-lines']), batch_size=batch_size, shuffle=True, num_workers=0) val_dataloader = torch.utils.data.DataLoader( ListDataset('your-test-lines'), batch_size=batch_size, shuffle=False, num_workers=0) model = make_model(len(char2token)) model.load_state_dict(torch.load('your-pretrain-model-path')) model.cuda() criterion = LabelSmoothing(size=len(char2token), padding_idx=0, smoothing=0.1) criterion.cuda() model_opt = NoamOpt( model.tgt_embed[0].d_model, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(10000): model.train() run_epoch(train_dataloader, model, SimpleLossCompute(model.generator, criterion, model_opt)) model.eval() test_loss = run_epoch( val_dataloader, model, SimpleLossCompute(model.generator, criterion, None)) print("test_loss", test_loss) torch.save(model.state_dict(), 'checkpoint/%08d_%f.pth' % (epoch, test_loss))
def synthetic_data(): print("synthetic_data task") # Train the simple copy task. V = 11 criterion = model_help.LabelSmoothing(size=V, padding_idx=0, smoothing=0.1) criterion.cuda() model = model_help.make_model(V, V, N=2) model.cuda() model_opt = model_help.NoamOpt( model.src_embed[0].d_model, 1, 400, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(10): model.train() run_epoch( data_help.data_gen(V, 30, 20), model, model_help.SimpleLossCompute(model.generator, criterion, args.device, model_opt)) model.eval() eval_loss = run_epoch( data_help.data_gen(V, 30, 5), model, model_help.SimpleLossCompute(model.generator, criterion, args.device, None)) print("eval loss: %f" % eval_loss.numpy()) model.eval() src = Variable(torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]])) src_mask = Variable(torch.ones(1, 1, 10)) print( greedy_decode(model, src.to(args.device), src_mask.to(args.device), max_len=10, start_symbol=1))
def main(): os.makedirs('checkpoint', exist_ok=True) V = 11 num_epochs = 10 batch_size = 30 train_batches = 20 test_batches = 5 criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0) model = make_model(V, V, N=2).to(device) model_opt = NoamOpt( model.src_embed[0].d_model, 1, 400, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) for epoch in range(num_epochs): # training model.train() train_loss = run_epoch( data_gen(V, batch_size=batch_size, nbatches=train_batches), model, SimpleLossCompute(model.generator, criterion, model_opt)) experiment.log_metric('train_loss', train_loss, step=epoch) # validation model.eval() valid_loss = run_epoch( data_gen(V, batch_size=30, nbatches=test_batches), model, SimpleLossCompute(model.generator, criterion, None)) experiment.log_metric('valid_loss', valid_loss, step=epoch) print('valid_loss:', valid_loss) torch.save(model.state_dict(), 'checkpoint/model.pt')
def main(): # Get the data data_train = pd.read_csv('dataset/train.csv') data_test = pd.read_csv('dataset/test.csv') # Transforming and dividing features Id_test = data_test['PassengerId'] selected_features = ['Pclass','Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked'] df_train, df_test = data_process.transform_features(data_train, data_test, selected_features) df_train, df_test = data_process.features_scaling(df_train, df_test, selected_features) X_train, Y_train, X_test, Y_test, test_X = data_process.split_data(df_train, df_test, selected_features) # Set parameters parameters = {} parameters['model_path'] = 'model/Titanic.ckpt' parameters['n_input'], parameters['n_features'] = X_train.shape parameters['n_hidden'] = 2 parameters['hidden_dim'] = 40 parameters['n_class'] = 1 parameters['learning_rate'] = 0.01 parameters['training_epochs'] = 15000 parameters['visualize'] = False if ((len(argv) > 1 and argv[1] == '-v') or (len(argv) > 2 and argv[2] == '-v')): parameters['visualize'] = True # Get model & train titanic_model = model.make_model(parameters) if (len(argv) > 1 and argv[1] == '-n') or (len(argv) > 2 and argv[2] == '-n'): model.neural_network(X_train, Y_train, parameters, titanic_model, X_test, Y_test) # Print accuracy if os.path.isfile(parameters['model_path']) == True: accuracy_estimation.Accuracy(parameters, titanic_model, X_train, Y_train, X_test, Y_test) # Output the submission to estimation.csv if os.path.isfile(parameters['model_path']) == True: accuracy_estimation.Estimation(parameters, titanic_model, test_X, Id_test) else: print("\nNo model found, please create a new file named 'Titanic.ckpt' in a directory named 'model' and launch the programme with th folowing commande :\n'python3 main.py -n'\n")
def train(train_data, train_labels, teacher_name, file_name, NUM_EPOCHS=50, BATCH_SIZE=128, TRAIN_TEMP=1): # Step 1: train the teacher model train_baseline.train(train_data, train_labels, teacher_name, NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP) # Step 2: evaluate the model on the training data at the training temperature soft_labels = np.zeros(train_labels.shape) with tf.Session() as s: model = make_model(teacher_name) xs = tf.placeholder(tf.float32, shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)) predictions = tf.nn.softmax(model(xs) / TRAIN_TEMP) for i in range(0, len(train_data), 100): predicted = predictions.eval({xs: train_data[i:i + 100]}) soft_labels[i:i + 100] = predicted # Step 3: train the distilled model on the new training data train_baseline.train(train_data, soft_labels, file_name, NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP)
def test(): opt = parse_args() opt.experiment = os.path.join(root_dir, opt.experiment) opt.chkpt = os.path.join(opt.experiment, opt.chkpt) opt.out_file = os.path.join(opt.experiment, opt.out_file) opt.out_json = os.path.join(opt.experiment, opt.out_json) sessions = json.loads(open(opt.test_file).read())['sessions'] # Model loading model = make_model(len(opt.word2idx)) chkpt = torch.load(opt.chkpt, map_location = lambda storage, log: storage) model.load_state_dict(chkpt) if opt.gpuid >= 0: model = model.cuda() # ====== *********************** ================ model.eval() # =============================================== # decode results = [] print('Decoding ...') decode_sessions = {'sessions': []} for session in sessions: n_session = {} n_session['session-id'] = session['session-id'] n_session['turns'] = [] for turn in session['turns']: asr_hyps = turn['asr-hyps'] asr_hyp = asr_hyps[0]['asr-hyp'] string = translate(model, asr_hyp, opt.word2idx, opt.idx2word, opt.cuda) if string == '': classes = [] else: classes = trp_reverse_process(string, 1) results.append((asr_hyp, string)) slu_hyp = [slot2dic(string) for string in classes] n_session['turns'].append( { 'asr-hyps': asr_hyps, 'slu-hyps': [{'slu-hyp': slu_hyp, 'score': 1.0}] } ) decode_sessions['sessions'].append(n_session) string = json.dumps(decode_sessions, sort_keys=True, indent=4, separators=(',', ':')) with open(opt.out_json, 'w') as f: f.write(string) print('Decode results saved in {}'.format(opt.save_file)) with open(opt.out_file, 'w') as f: for (enc, dec) in results: f.write('{}\t<=>\t{}\n'.format(enc.strip(), dec.strip()))
def train(): print("Loading data...") SRC, TGT, train, val, test = generate_dataloaders() devices = [0, 1, 2, 3] pad_idx = TGT.vocab.stoi["<blank>"] print("Making model...") model = make_model(len(SRC.vocab), len(TGT.vocab), N=6) model.cuda() criterion = LabelSmoothing( size=len(TGT.vocab), padding_idx=pad_idx, smoothing=0.1) criterion.cuda() BATCH_SIZE = 12000 train_iter = BatchIterator(train, batch_size=BATCH_SIZE, device=torch.device(0), repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)), batch_size_fn=batch_size_fn, train=True) valid_iter = BatchIterator(val, batch_size=BATCH_SIZE, device=torch.device(0), repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)), batch_size_fn=batch_size_fn, train=False) model_par = nn.DataParallel(model, device_ids=devices) model_opt = NoamOpt(model.src_embed[0].d_model, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) folder = get_unique_folder("./models/", "model") if not(os.path.exists(folder)): os.mkdir(folder) for epoch in tqdm(range(10)): model_par.train() run_epoch((rebatch(pad_idx, b) for b in train_iter), model_par, MultiGPULossCompute(model.generator, criterion, devices=devices, opt=model_opt)) model_par.eval() loss = run_epoch((rebatch(pad_idx, b) for b in valid_iter), model_par, MultiGPULossCompute(model.generator, criterion, devices=devices, opt=None)) torch.save(model.state_dict, os.path.join(folder, "model.bin." + str(epoch))) print(loss) for i, batch in enumerate(valid_iter): src = batch.src.transpose(0, 1)[:1] src_mask = (src != SRC.vocab.stoi["<blank>"]).unsqueeze(-2) out = greedy_decode(model, src, src_mask, max_len=60, start_symbol=TGT.vocab.stoi["<s>"]) print("Translation:", end="\t") for i in range(1, out.size(1)): sym = TGT.vocab.itos[out[0, i]] if sym == "</s>": break print(sym, end=" ") print() print("Target:", end="\t") for i in range(1, batch.trg.size(0)): sym = TGT.vocab.itos[batch.trg.data[i, 0]] if sym == "</s>": break print(sym, end=" ") print() break
def follower(): new_model = make_model() dream_model = make_model() while 1: # print('waiting for packet') packet = comm.recv(source=0) #comm.Recv(packet, source=0) current_env_name = packet['current_env_name'] dream_mode = packet['dream_mode'] packet = packet['result'] assert(len(packet) == SOLUTION_PACKET_SIZE), (len(packet), SOLUTION_PACKET_SIZE) solutions = decode_solution_packet(packet) results = [] if dream_mode: new_model.make_env(current_env_name + '_dream', model = dream_model) else: new_model.make_env(current_env_name) for i, solution in enumerate(solutions): # print(f'working on solution {i+1} of {len(solutions)}') worker_id, jobidx, seed, max_len, weights = solution worker_id = int(worker_id) possible_error = "work_id = " + str(worker_id) + " rank = " + str(rank) assert worker_id == rank, possible_error jobidx = int(jobidx) seed = int(seed) fitness, timesteps = worker(weights, seed, max_len, new_model) results.append([worker_id, jobidx, fitness, timesteps]) new_model.env.close() result_packet = encode_result_packet(results) assert len(result_packet) == RESULT_PACKET_SIZE comm.Send(result_packet, dest=0)
def __init__(self): super().__init__() self.net = make_model(conf["model"]).to(device=device) self.net.load_weights(args) self.renderer = NeRFRenderer.from_conf( conf["renderer"], white_bkgd=not args.black, eval_batch_size=args.ray_batch_size).to(device=device)
def train(train_data, train_labels, file_name, NUM_EPOCHS=50, BATCH_SIZE=128, TRAIN_TEMP=1): train_size = train_labels.shape[0] train_xs = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)) train_ys = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS)) model, saver = make_model(None, NUM_CHANNELS=NUM_CHANNELS, IMAGE_SIZE=IMAGE_SIZE) logits = model(train_xs, True) loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits/TRAIN_TEMP, labels=train_ys)) lr = tf.placeholder(tf.float32, []) momentum = tf.placeholder(tf.float32, []) optimizer = tf.train.MomentumOptimizer(lr, momentum).minimize(loss) check_xs = tf.placeholder(tf.float32, shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)) check_prediction = tf.nn.softmax(model(check_xs)) def check(data, labels): err = [] for i in range(0,len(data),100): predicted = check_prediction.eval({check_xs: data[i:i+100]}) valids = np.argmax(predicted, 1) == np.argmax(labels[i:i+100], 1) err.extend(valids) return 1-np.mean(err) with tf.Session() as s: tf.global_variables_initializer().run() for step in range(NUM_EPOCHS * train_size // BATCH_SIZE): epoch = (float(step) * BATCH_SIZE / train_size) offset = np.random.random_integers(0, train_size-1, BATCH_SIZE) batch_data = train_data[offset, :, :, :] batch_labels = train_labels[offset, :] feed_dict = {train_xs: batch_data, train_ys: batch_labels, lr: (0.5**int(epoch/10))*0.01, momentum: (0.5**int(epoch/10))*0.9} s.run(optimizer, feed_dict=feed_dict) if step % 100 == 0: print("Step %d/%d"%(step, NUM_EPOCHS * train_size // BATCH_SIZE)) print("Validation error: ", check(validation_data, validation_labels)) print("\nTest error: ", check(test_data, test_labels)) saver(s, file_name)
def __init__(self, cfg): self.model = make_model(cfg, 751) self.model.load_param(cfg.TEST_WEIGHT) self.device = "cuda" if self.device: if torch.cuda.device_count() > 1: self.model = nn.DataParallel(self.model) self.model.to(self.device) self.model.eval()
def sample(args): # load ascii text and covert to lowercase with codecs.open(os.path.join(args.data_dir, 'input.txt'), "r", encoding='utf-8') as f: raw_text = f.read() # create mapping of unique chars to integers, and a reverse mapping chars = sorted(list(set(raw_text))) char_to_int = dict((c, i) for i, c in enumerate(chars)) int_to_char = dict((i, c) for i, c in enumerate(chars)) # summarize the loaded data n_chars = len(raw_text) n_vocab = len(chars) print("Total Characters: ", n_chars) print("Total Vocab: ", n_vocab) # prepare the dataset of input to output pairs encoded as integers seq_length = 100 dataX = [] dataY = [] for i in range(0, n_chars - seq_length, 1): seq_in = raw_text[i:i + seq_length] seq_out = raw_text[i + seq_length] dataX.append([char_to_int[char] for char in seq_in]) dataY.append(char_to_int[seq_out]) n_patterns = len(dataX) print("Total Patterns: "), n_patterns X = numpy.reshape(dataX, (n_patterns, seq_length, 1)) # normalize X = X / float(n_vocab) # one hot encode the output variable y = np_utils.to_categorical(dataY) # load the network weights model = make_model(X, y) iteration = 10 filename = find_file( 'snapshot_' + MODEL_NAME + '_{:02d}'.format(iteration) + '*.hdf5', 'models') model.load_weights(filename) model.compile(loss='categorical_crossentropy', optimizer='adam') # pick a random seed start = numpy.random.randint(0, len(dataX) - 1) pattern = dataX[start] print("Seed:") print("\"", ''.join([int_to_char[value] for value in pattern]), "\"") # generate characters for i in range(1000): x = numpy.reshape(pattern, (1, len(pattern), 1)) x = x / float(n_vocab) prediction = model.predict(x, verbose=0) index = numpy.argmax(prediction) result = int_to_char[index] seq_in = [int_to_char[value] for value in pattern] sys.stdout.write(result) pattern.append(index) pattern = pattern[1:len(pattern)] print("\nDone.")
def get_models(args): ae_model = get_cuda(make_model(d_vocab=args.vocab_size, N=args.num_layers_AE, d_model=args.transformer_model_size, latent_size=args.latent_size, d_ff=args.transformer_ff_size)) dis_model = get_cuda(Classifier(latent_size=args.latent_size, output_size=args.label_size)) ae_model.load_state_dict(torch.load(args.current_save_path + 'ae_model_params.pkl', map_location=torch.device('cpu'))) dis_model.load_state_dict(torch.load(args.current_save_path + 'dis_model_params.pkl', map_location=torch.device('cpu'))) return ae_model, dis_model
def one_sentence_translate(sent, beam_search=True): # 初始化模型 model = make_model(config.src_vocab_size, config.tgt_vocab_size, config.n_layers, config.d_model, config.d_ff, config.n_heads, config.dropout) BOS = english_tokenizer_load().bos_id() # 2 EOS = english_tokenizer_load().eos_id() # 3 src_tokens = [[BOS] + english_tokenizer_load().EncodeAsIds(sent) + [EOS]] batch_input = torch.LongTensor(np.array(src_tokens)).to(config.device) translate(batch_input, model, use_beam=beam_search)
def main(): V = 11 model = make_model(V, V, N=2).to(device) model.load_state_dict(torch.load('checkpoint/model.pt')) model.to(device) model.eval() src = torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]).to(device) src_mask = torch.ones(1, 1, 10).to(device) result = greedy_decode(model, src, src_mask, max_len=10, start_symbol=1) print(result)
def make_model_higher(): w_conv = randn(config.frame_out, config.frame_len, requires_grad=config.conv_deconv_grad) if config.init_fourier: with no_grad(): for f in range(config.frame_out): w_conv[f, ...] = cos(2 * pi * (f + 1) / config.frame_len * arange(0, config.frame_len, 1)) convolver = FF(w_conv) else: if config.init_xavier: xavier_normal_(w_conv, gain=5 / 3) convolver = FFT(w_conv) if config.conv_deconv_same: deconvolver = convolver else: if config.init_fourier: w_deconv = w_conv.detach() w_deconv.requires_grad = config.conv_deconv_grad deconvolver = FF(w_deconv) else: w_deconv = randn(config.frame_out, config.frame_len, requires_grad=config.conv_deconv_grad) if config.init_xavier: xavier_normal_(w_deconv, gain=5 / 3) deconvolver = FFT(w_deconv) convolver = [convolver] deconvolver = [deconvolver] # body = config.creation_info[1:-1] enc = make_model(config.attention1_info) dec = make_model(config.attention2_info) return [convolver, enc, dec, deconvolver]
def setup_embeddings(script, config): # Get parameters num_vertices = config.getint('Graph', 'num_vertices') motif_size = config.getint('Motifs', 'motif_size') walk_length = config.getint('Walks', 'walk_length') embeddings_dir = config.get('Embeddings', 'embeddings_dir') embed_dim = config.getint('Embeddings', 'embed_dim') learn_rate = config.getfloat('Embeddings', 'learn_rate') mini_batch_size = config.getint('Embeddings', 'mini_batch_size') sgd_steps = config.getint('Embeddings', 'sgd_steps') sgd_steps_per_epoch = config.getint('Embeddings', 'sgd_steps_per_epoch') assert (num_vertices>0 and motif_size>0 and walk_length>=motif_size and embeddings_dir and embed_dim>0 and mini_batch_size>0 and sgd_steps>=0 and sgd_steps_per_epoch>0), \ 'invalid configuration for training embeddings' # Construct LBANN objects num_epochs = (sgd_steps + sgd_steps_per_epoch - 1) // sgd_steps_per_epoch trainer = lbann.Trainer( mini_batch_size=mini_batch_size, num_parallel_readers=0, ) model_ = make_model( motif_size, walk_length, num_vertices, embed_dim, learn_rate, num_epochs, embeddings_dir, ) optimizer = lbann.SGD(learn_rate=learn_rate) data_reader = make_data_reader() # Add LBANN invocation to batch script prototext_file = os.path.join(script.work_dir, 'experiment.prototext') lbann.proto.save_prototext( prototext_file, trainer=trainer, model=model_, data_reader=data_reader, optimizer=optimizer, ) script.add_body_line('') script.add_body_line('# Train embeddings') script.add_parallel_command([ lbann.lbann_exe(), f'--prototext={prototext_file}', f'--num_io_threads=1', ])
def check_opt(): """check learning rate changes""" import numpy as np import matplotlib.pyplot as plt model = make_model(config.src_vocab_size, config.tgt_vocab_size, config.n_layers, config.d_model, config.d_ff, config.n_heads, config.dropout) opt = get_std_opt(model) # Three settings of the lrate hyperparameters. opts = [opt, NoamOpt(512, 1, 20000, None), NoamOpt(256, 1, 10000, None)] plt.plot(np.arange(1, 50000), [[opt.rate(i) for opt in opts] for i in range(1, 50000)]) plt.legend(["512:10000", "512:20000", "256:10000"]) plt.show()
def translate(): # Initialization opt = parse_args() src_word2idx = torch.load(opt.vocab)['src'] tgt_word2idx = torch.load(opt.vocab)['tgt'] print("src_vocab: {}, tgt_vocab: {}".format(len(src_word2idx), len(tgt_word2idx))) # Model definition model = make_model(len(src_word2idx), len(tgt_word2idx)) translator = Translator(model, opt.beam_size, opt.n_best, opt.max_length) translator.translate(opt)
def train(train_data, train_labels, teacher_name, file_name, NUM_EPOCHS=50, BATCH_SIZE=128, TRAIN_TEMP=1): # Step 1: train the teacher model train_baseline.train(train_data, train_labels, teacher_name, NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP) # Step 2: evaluate the model on the training data at the training temperature soft_labels = np.zeros(train_labels.shape) with tf.Session() as s: model = make_model(teacher_name) xs = tf.placeholder(tf.float32, shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)) predictions = tf.nn.softmax(model(xs)/TRAIN_TEMP) for i in range(0,len(train_data),100): predicted = predictions.eval({xs: train_data[i:i+100]}) soft_labels[i:i+100] = predicted # Step 3: train the distilled model on the new training data train_baseline.train(train_data, soft_labels, file_name, NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP)
def slave(): new_model = make_model() while 1: #print('waiting for packet') packet = comm.recv(source=0) #comm.Recv(packet, source=0) current_env_name = packet['current_env_name'] packet = packet['result'] assert(len(packet) == SOLUTION_PACKET_SIZE) solutions = decode_solution_packet(packet) results = [] #tracker2 = SummaryTracker() new_model.make_env(current_env_name) #tracker2.print_diff() for solution in solutions: worker_id, jobidx, seed, train_mode, max_len, weights = solution assert (train_mode == 1 or train_mode == 0), str(train_mode) worker_id = int(worker_id) possible_error = "work_id = " + str(worker_id) + " rank = " + str(rank) assert worker_id == rank, possible_error jobidx = int(jobidx) seed = int(seed) fitness, timesteps = worker(weights, seed, max_len, new_model, train_mode) results.append([worker_id, jobidx, fitness, timesteps]) new_model.env.close() result_packet = encode_result_packet(results) assert len(result_packet) == RESULT_PACKET_SIZE comm.Send(result_packet, dest=0)
solver = KOSolver(k=k, n_t=n_t) # Collect data x = best.design.latin_center(num_samples, k) Y = [] for i in range(num_samples): Y.append(solver(x[i, :])) X = (x, solver.X_fixed[0]) H = tuple([np.ones((x.shape[0], 1)) for x in X]) Y = np.vstack(Y) # Save the data for later use data_file = 'ko_data_s=%d.pickle' % num_samples with open(data_file, 'wb') as fd: pickle.dump((X, H, Y), fd, pickle.HIGHEST_PROTOCOL) cgp_model = model.make_model(X, Y) smc_sampler = best.smc.SMC(cgp_model, num_particles=num_particles, num_mcmc=num_mcmc, verbose=1, gamma_is_an_exponent=True, mpi=mpi) smc_sampler.initialize(0.) pa = smc_sampler.get_particle_approximation() smc_sampler.move_to(1.) pa = smc_sampler.get_particle_approximation() # dump the result to a file to process later gpa = pa.allgather() if mpi.COMM_WORLD.Get_rank() == 0: out_file = 'ko_smc_s=%d_p=%d_m=%d.pickle' %(num_samples, num_particles, num_mcmc)
from model import * # from mpl_toolkits.basemap import Basemap # from matplotlib import * from pylab import * from numpy import * import model import pymc as pm data = csv2rec('duffy-jittered.csv') # Convert from record array to dictionary data = dict([(k,data[k]) for k in data.dtype.names]) # Create model. Use the HDF5 database backend, because the trace will take up a lot of memory. DuffySampler = pm.MCMC(model.make_model(**data), db='hdf5', dbcomplevel=1, dbcomplib='zlib', dbname='duffydb.hdf5') # ==================== # = Do the inference = # ==================== # Use GPEvaluationGibbs step methods. DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_b, DuffySampler.V_b, DuffySampler.tilde_fb) DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_s, DuffySampler.V_s, DuffySampler.tilde_fs) # Run the MCMC. DuffySampler.isample(50000,10000,100) n = len(DuffySampler.trace('V_b')[:]) # ========================== # = Mean and variance maps =
import model import best import pymc import matplotlib.pyplot as plt if __name__ == '__main__': # Set the sampling parameters k = 1 # Number of dimensions n_t = 10 # Number of time steps num_samples = 20 # Number of samples # Initialize the solver solver = KOSolver(k=k, n_t=n_t) # Collect data X = best.design.latin_center(num_samples, k) Y = [] for i in range(num_samples): Y.append(solver(X[i, :])) Y = np.vstack(Y) cgp_model = model.make_model((X, solver.X_fixed[0]), Y) mcmc_sampler = pymc.MCMC(cgp_model) mcmc_sampler.sample(100000, thin=1000, burn=10000) pymc.Matplot.plot(mcmc_sampler) plt.show()
def initialize_settings(sigma_init=0.1, sigma_decay=0.9999, init_opt = ''): global population, filebase, controller_filebase, model, num_params, es, PRECISION, SOLUTION_PACKET_SIZE, RESULT_PACKET_SIZE population = num_worker * num_worker_trial filebase = './log/'+env_name+'.'+optimizer+'.'+str(num_episode)+'.'+str(population) controller_filebase = './controller/'+env_name+'.'+optimizer+'.'+str(num_episode)+'.'+str(population) model = make_model() num_params = model.param_count #print("size of model", num_params) if len(init_opt) > 0: es = pickle.load(open(init_opt, 'rb')) else: if optimizer == 'ses': ses = PEPG(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_alpha=0.2, sigma_limit=0.02, elite_ratio=0.1, weight_decay=0.005, popsize=population) es = ses elif optimizer == 'ga': ga = SimpleGA(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_limit=0.02, elite_ratio=0.1, weight_decay=0.005, popsize=population) es = ga elif optimizer == 'cma': cma = CMAES(num_params, sigma_init=sigma_init, popsize=population) es = cma elif optimizer == 'pepg': pepg = PEPG(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_alpha=0.20, sigma_limit=0.02, learning_rate=0.01, learning_rate_decay=1.0, learning_rate_limit=0.01, weight_decay=0.005, popsize=population) es = pepg else: oes = OpenES(num_params, sigma_init=sigma_init, sigma_decay=sigma_decay, sigma_limit=0.02, learning_rate=0.01, learning_rate_decay=1.0, learning_rate_limit=0.01, antithetic=antithetic, weight_decay=0.005, popsize=population) es = oes PRECISION = 10000 SOLUTION_PACKET_SIZE = (5+num_params)*num_worker_trial RESULT_PACKET_SIZE = 4*num_worker_trial
from model import * # from mpl_toolkits.basemap import Basemap # from matplotlib import * from pylab import * from numpy import * import model import pymc as pm data = csv2rec("duffy-jittered.csv") # Convert from record array to dictionary data = dict([(k, data[k]) for k in data.dtype.names]) # Create model. Use the HDF5 database backend, because the trace will take up a lot of memory. DuffySampler = pm.MCMC(model.make_model(**data), db="hdf5", dbcomplevel=1, dbcomplib="zlib", dbname="duffydb.hdf5") # ==================== # = Do the inference = # ==================== # Use GPEvaluationGibbs step methods. DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_b, DuffySampler.V_b, DuffySampler.tilde_fb) DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_s, DuffySampler.V_s, DuffySampler.tilde_fs) # Run the MCMC. DuffySampler.isample(50000, 10000, 100) n = len(DuffySampler.trace("V_b")[:]) # ========================== # = Mean and variance maps =
import sys import time import tensorflow as tf import numpy as np from model import make_model, preprocess from setup import * def compute(dat,lab): BATCH_SIZE = 100 img = tf.Variable(np.zeros((BATCH_SIZE,IMAGE_SIZE,IMAGE_SIZE,NUM_CHANNELS),dtype=np.float32)) out = model(img) r = [] for i in range(0,len(dat),BATCH_SIZE): data = img o = s.run(out, feed_dict={img: preprocess(dat[i:i+BATCH_SIZE])}) r.extend(o) print(np.mean(np.abs(r)),np.std(np.abs(r))) with tf.Session() as s: model = make_model(sys.argv[1],s) compute(test_data, test_labels)