def train(train_loader, num_classes):
parser = argparse.ArgumentParser(description="ReID Baseline Training")
parser.add_argument(
"--config_file", default="", help="path to config file", type=str
)
parser.add_argument("opts", help="Modify config options using the command-line", default=None,
nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.config_file != "":
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
set_seed(cfg.SOLVER.SEED)
output_dir = cfg.OUTPUT_DIR
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
logger = setup_logger("reid_baseline", output_dir, if_train=True)
logger.info("Saving model in the path :{}".format(cfg.OUTPUT_DIR))
logger.info(args)
if args.config_file != "":
logger.info("Loaded configuration file {}".format(args.config_file))
with open(args.config_file, 'r') as cf:
config_str = "\n" + cf.read()
logger.info(config_str)
logger.info("Running with config:\n{}".format(cfg))
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.MODEL.DEVICE_ID
if cfg.MODEL.PRETRAIN_CHOICE == 'finetune':
model = make_model(cfg, num_class=num_classes)
model.load_param_finetune(cfg.MODEL.PRETRAIN_PATH)
print('Loading pretrained model for finetuning......')
else:
model = make_model(cfg, num_class=num_classes)
loss_func = make_loss(cfg, num_classes=num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = WarmupCosineAnnealingLR(optimizer, cfg.SOLVER.MAX_EPOCHS, cfg.SOLVER.DELAY_ITERS, cfg.SOLVER.ETA_MIN_LR,
cfg.SOLVER.WARMUP_FACTOR, cfg.SOLVER.WARMUP_EPOCHS, cfg.SOLVER.WARMUP_METHOD)
logger.info("use WarmupCosineAnnealingLR, delay_step:{}".format(cfg.SOLVER.DELAY_ITERS))
do_train(
cfg,
model,
train_loader,
optimizer,
scheduler, # modify for using self trained model
loss_func
)
def check_video_face():
if request.method == "POST":
face_capture.perform()
model.make_model()
return render_template('video_kyc.html',
context="Model created Successfully")
else:
return render_template('video_kyc.html', context='')
def test_make_model(self):
model = make_model()
model_3_topics = make_model(3)
self.assertEqual(model.n_components, 5,
"False default num of components")
self.assertEqual(model_3_topics.n_components, 3,
"False num of components passed to model")
self.assertEqual(model.solver, "mu",
"False solver in model")
self.assertEqual(model.beta_loss, "kullback-leibler",
"False kind of beta loss")
self.assertEqual(model.alpha, 0.1,
"False alpha value")
def main(disp_text=True):
if config.fresh_model:
config.all_losses = []
save_model(make_model())
model = load_model()
if disp_text: print('created model.', end=' ')
else:
model = load_model()
if not model:
save_model(make_model())
model = load_model()
if disp_text: print('created model.', end=' ')
else:
if disp_text: print('loaded model.', end=' ')
data = load_data()
data, data_dev = split_data(data)
data = [d for i,d in enumerate(data) if i in [8,10,13,14]]
print()
seq_lens = [len(d) for d in data]
print(f'seq lens: {seq_lens}')
min_seq_len = min(seq_lens)
print(f'min seq len: {min_seq_len}')
if not config.max_seq_len or config.max_seq_len > min_seq_len:
config.max_seq_len = min_seq_len
data = [d[:config.max_seq_len] for d in data]
# from random import choice
# from torch import randn
# data = [[randn(config.in_size) for _ in range(choice(range(config.max_seq_len//2,config.max_seq_len)))] for _ in range(10)]
# data_dev = []
# for d in data: print(len(d))
if not config.batch_size or config.batch_size >= len(data):
config.batch_size = len(data)
elif config.batch_size < 1:
config.batch_size = int(len(data)*config.batch_size)
if disp_text: print(f'hm data: {len(data)}, hm dev: {len(data_dev)}, bs: {config.batch_size}, lr: {config.learning_rate}, \ntraining started @ {now()}')
for ep in range(config.hm_epochs):
for i, batch in enumerate(batchify_data(data)):
train_on(model, batch)
return model
def train(params):
batch_size = 64
train_dataloader = torch.utils.data.DataLoader(ListDataset(['data/dataset/train.txt']), batch_size=batch_size,
shuffle=True, num_workers=0)
val_dataloader = torch.utils.data.DataLoader(ListDataset('data/dataset/validation.txt'), batch_size=batch_size,
shuffle=False,
num_workers=0)
model = make_model(len(char2token))
model.to(device=params['device'])
criterion = LabelSmoothing(size=len(char2token), padding_idx=0, smoothing=0.1)
criterion.to(device=params['device'])
model_opt = NoamOpt(model.tgt_embed[0].d_model, 1, 2000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
for epoch in range(10000):
params['epoch'] = epoch
model.train()
train_loss = run_epoch(train_dataloader, model,
SimpleLossCompute(model.generator, criterion, model_opt), params)
print("train_loss", train_loss)
model.eval()
val_loss = run_epoch(val_dataloader, model,
SimpleLossCompute(model.generator, criterion, None), params)
print("val_loss", val_loss)
torch.save(model.state_dict(), 'checkpoint/%08d_%f.pth' % (epoch, val_loss))
def slave():
#packet = (np.empty(SOLUTION_PACKET_SIZE, dtype=np.int32),'')
while 1:
packet = comm.recv(source=0)
current_env_name = packet['current_env_name']
packet = packet['result']
#packet, current_env_name = packet
assert (len(packet) == SOLUTION_PACKET_SIZE)
solutions = decode_solution_packet(packet)
results = []
new_model = make_model()
new_model.make_env(current_env_name)
for solution in solutions:
worker_id, jobidx, seed, train_mode, max_len, weights = solution
assert (train_mode == 1 or train_mode == 0), str(train_mode)
worker_id = int(worker_id)
possible_error = "work_id = " + str(worker_id) + " rank = " + str(
rank)
assert worker_id == rank, possible_error
jobidx = int(jobidx)
seed = int(seed)
fitness, timesteps = worker(weights, seed, max_len, new_model,
train_mode)
results.append([worker_id, jobidx, fitness, timesteps])
result_packet = encode_result_packet(results)
assert len(result_packet) == RESULT_PACKET_SIZE
comm.Send(result_packet, dest=0)
def client():
new_model = make_model(sys.argv[1])
while True:
packet = comm.recv(source=0)
packet = packet['result']
solutions = decode_solution_packet(packet)
results = []
new_model.make_env()
for solution in solutions:
worker_id, jobidx, seed, train_mode, max_len, weights = solution
worker_id = int(worker_id)
jobidx = int(jobidx)
seed = int(seed)
fitness, timesteps = worker(weights, seed, max_len, new_model,
train_mode)
results.append([worker_id, jobidx, fitness, timesteps])
new_model.env.close()
result_packet = encode_result_packet(results)
comm.Send(result_packet, dest=0)
def init_model():
from setting import LAYERS, D_MODEL, D_FF, DROPOUT, H_NUM, TGT_VOCAB, SRC_VOCAB
from model import make_model
# 模型的初始化
model = make_model(SRC_VOCAB, TGT_VOCAB, LAYERS, D_MODEL, D_FF, H_NUM,
DROPOUT)
return model
def initialize_settings(sigma_init=0.1, sigma_decay=0.9999):
global population, filebase, game, model, num_params, es, PRECISION, SOLUTION_PACKET_SIZE, RESULT_PACKET_SIZE
population = num_worker * num_worker_trial
filebase = 'log/' + gamename + '.' + optimizer + '.' + str(
num_episode) + '.' + str(population)
game = config.games[gamename]
model = make_model(game)
num_params = model.param_count
print("size of model", num_params)
if optimizer == 'ses':
ses = PEPG(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_alpha=0.2,
sigma_limit=0.02,
elite_ratio=0.1,
weight_decay=0.005,
popsize=population)
es = ses
elif optimizer == 'ga':
ga = SimpleGA(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_limit=0.02,
elite_ratio=0.1,
weight_decay=0.005,
popsize=population)
es = ga
elif optimizer == 'cma':
cma = CMAES(num_params, sigma_init=sigma_init, popsize=population)
es = cma
elif optimizer == 'pepg':
pepg = PEPG(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_alpha=0.20,
sigma_limit=0.02,
learning_rate=0.01,
learning_rate_decay=1.0,
learning_rate_limit=0.01,
weight_decay=0.005,
popsize=population)
es = pepg
else:
oes = OpenES(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_limit=0.02,
learning_rate=0.01,
learning_rate_decay=1.0,
learning_rate_limit=0.01,
antithetic=antithetic,
weight_decay=0.005,
popsize=population)
es = oes
PRECISION = 10000
SOLUTION_PACKET_SIZE = (5 + num_params) * num_worker_trial
RESULT_PACKET_SIZE = 4 * num_worker_trial
def predict(load_path):
model = make_model()
model.load_weights(load_path + "weights_best.hdf5")
sgd = SGD(lr=0.002, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss="categorical_crossentropy",
metrics=['accuracy'])
for fold in clips:
print(fold)
for clip in clips[fold]:
print(clip)
x = []
x.append(get_features(clip, 0))
x = np.array(x)
clip_prediction = model.predict(x, 1)
for i in range(1, clip.PElog_spectra.shape[2]):
x = []
x.append(get_features(clip, i))
x = np.array(x)
clip_prediction *= model.predict(
x, 1) # multiply probabilities for the segments
print(np.argmax(prediction), clip.target)
print(clip_prediction[0][int(clip.target)])
def __init__(self, env, num_episodes, expert_file):
self.env = env
self.expert = tf.keras.models.load_model(expert_file)
self.num_episodes = num_episodes
print("Expert insput shape ", self.expert.input_shape)
self.model = make_model()
print("Model input shape ", self.model.input_shape)
def train():
batch_size = 64
train_dataloader = torch.utils.data.DataLoader(ListDataset(
['your-train-lines']),
batch_size=batch_size,
shuffle=True,
num_workers=0)
val_dataloader = torch.utils.data.DataLoader(
ListDataset('your-test-lines'),
batch_size=batch_size,
shuffle=False,
num_workers=0)
model = make_model(len(char2token))
model.load_state_dict(torch.load('your-pretrain-model-path'))
model.cuda()
criterion = LabelSmoothing(size=len(char2token),
padding_idx=0,
smoothing=0.1)
criterion.cuda()
model_opt = NoamOpt(
model.tgt_embed[0].d_model, 1, 2000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
for epoch in range(10000):
model.train()
run_epoch(train_dataloader, model,
SimpleLossCompute(model.generator, criterion, model_opt))
model.eval()
test_loss = run_epoch(
val_dataloader, model,
SimpleLossCompute(model.generator, criterion, None))
print("test_loss", test_loss)
torch.save(model.state_dict(),
'checkpoint/%08d_%f.pth' % (epoch, test_loss))
def synthetic_data():
print("synthetic_data task")
# Train the simple copy task.
V = 11
criterion = model_help.LabelSmoothing(size=V, padding_idx=0, smoothing=0.1)
criterion.cuda()
model = model_help.make_model(V, V, N=2)
model.cuda()
model_opt = model_help.NoamOpt(
model.src_embed[0].d_model, 1, 400,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
for epoch in range(10):
model.train()
run_epoch(
data_help.data_gen(V, 30, 20), model,
model_help.SimpleLossCompute(model.generator, criterion,
args.device, model_opt))
model.eval()
eval_loss = run_epoch(
data_help.data_gen(V, 30, 5), model,
model_help.SimpleLossCompute(model.generator, criterion,
args.device, None))
print("eval loss: %f" % eval_loss.numpy())
model.eval()
src = Variable(torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]))
src_mask = Variable(torch.ones(1, 1, 10))
print(
greedy_decode(model,
src.to(args.device),
src_mask.to(args.device),
max_len=10,
start_symbol=1))
def main():
os.makedirs('checkpoint', exist_ok=True)
V = 11
num_epochs = 10
batch_size = 30
train_batches = 20
test_batches = 5
criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)
model = make_model(V, V, N=2).to(device)
model_opt = NoamOpt(
model.src_embed[0].d_model, 1, 400,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
for epoch in range(num_epochs):
# training
model.train()
train_loss = run_epoch(
data_gen(V, batch_size=batch_size, nbatches=train_batches), model,
SimpleLossCompute(model.generator, criterion, model_opt))
experiment.log_metric('train_loss', train_loss, step=epoch)
# validation
model.eval()
valid_loss = run_epoch(
data_gen(V, batch_size=30, nbatches=test_batches), model,
SimpleLossCompute(model.generator, criterion, None))
experiment.log_metric('valid_loss', valid_loss, step=epoch)
print('valid_loss:', valid_loss)
torch.save(model.state_dict(), 'checkpoint/model.pt')
def main():
# Get the data
data_train = pd.read_csv('dataset/train.csv')
data_test = pd.read_csv('dataset/test.csv')
# Transforming and dividing features
Id_test = data_test['PassengerId']
selected_features = ['Pclass','Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
df_train, df_test = data_process.transform_features(data_train, data_test, selected_features)
df_train, df_test = data_process.features_scaling(df_train, df_test, selected_features)
X_train, Y_train, X_test, Y_test, test_X = data_process.split_data(df_train, df_test, selected_features)
# Set parameters
parameters = {}
parameters['model_path'] = 'model/Titanic.ckpt'
parameters['n_input'], parameters['n_features'] = X_train.shape
parameters['n_hidden'] = 2
parameters['hidden_dim'] = 40
parameters['n_class'] = 1
parameters['learning_rate'] = 0.01
parameters['training_epochs'] = 15000
parameters['visualize'] = False
if ((len(argv) > 1 and argv[1] == '-v') or (len(argv) > 2 and argv[2] == '-v')):
parameters['visualize'] = True
# Get model & train
titanic_model = model.make_model(parameters)
if (len(argv) > 1 and argv[1] == '-n') or (len(argv) > 2 and argv[2] == '-n'):
model.neural_network(X_train, Y_train, parameters, titanic_model, X_test, Y_test)
# Print accuracy
if os.path.isfile(parameters['model_path']) == True:
accuracy_estimation.Accuracy(parameters, titanic_model, X_train, Y_train, X_test, Y_test)
# Output the submission to estimation.csv
if os.path.isfile(parameters['model_path']) == True:
accuracy_estimation.Estimation(parameters, titanic_model, test_X, Id_test)
else:
print("\nNo model found, please create a new file named 'Titanic.ckpt' in a directory named 'model' and launch the programme with th folowing commande :\n'python3 main.py -n'\n")
def train(train_data,
train_labels,
teacher_name,
file_name,
NUM_EPOCHS=50,
BATCH_SIZE=128,
TRAIN_TEMP=1):
# Step 1: train the teacher model
train_baseline.train(train_data, train_labels, teacher_name, NUM_EPOCHS,
BATCH_SIZE, TRAIN_TEMP)
# Step 2: evaluate the model on the training data at the training temperature
soft_labels = np.zeros(train_labels.shape)
with tf.Session() as s:
model = make_model(teacher_name)
xs = tf.placeholder(tf.float32,
shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
predictions = tf.nn.softmax(model(xs) / TRAIN_TEMP)
for i in range(0, len(train_data), 100):
predicted = predictions.eval({xs: train_data[i:i + 100]})
soft_labels[i:i + 100] = predicted
# Step 3: train the distilled model on the new training data
train_baseline.train(train_data, soft_labels, file_name, NUM_EPOCHS,
BATCH_SIZE, TRAIN_TEMP)
def test():
opt = parse_args()
opt.experiment = os.path.join(root_dir, opt.experiment)
opt.chkpt = os.path.join(opt.experiment, opt.chkpt)
opt.out_file = os.path.join(opt.experiment, opt.out_file)
opt.out_json = os.path.join(opt.experiment, opt.out_json)
sessions = json.loads(open(opt.test_file).read())['sessions']
# Model loading
model = make_model(len(opt.word2idx))
chkpt = torch.load(opt.chkpt, map_location = lambda storage, log: storage)
model.load_state_dict(chkpt)
if opt.gpuid >= 0:
model = model.cuda()
# ====== *********************** ================
model.eval()
# ===============================================
# decode
results = []
print('Decoding ...')
decode_sessions = {'sessions': []}
for session in sessions:
n_session = {}
n_session['session-id'] = session['session-id']
n_session['turns'] = []
for turn in session['turns']:
asr_hyps = turn['asr-hyps']
asr_hyp = asr_hyps[0]['asr-hyp']
string = translate(model, asr_hyp, opt.word2idx, opt.idx2word, opt.cuda)
if string == '':
classes = []
else:
classes = trp_reverse_process(string, 1)
results.append((asr_hyp, string))
slu_hyp = [slot2dic(string) for string in classes]
n_session['turns'].append(
{
'asr-hyps': asr_hyps,
'slu-hyps': [{'slu-hyp': slu_hyp, 'score': 1.0}]
}
)
decode_sessions['sessions'].append(n_session)
string = json.dumps(decode_sessions, sort_keys=True, indent=4, separators=(',', ':'))
with open(opt.out_json, 'w') as f:
f.write(string)
print('Decode results saved in {}'.format(opt.save_file))
with open(opt.out_file, 'w') as f:
for (enc, dec) in results:
f.write('{}\t<=>\t{}\n'.format(enc.strip(), dec.strip()))
def train():
print("Loading data...")
SRC, TGT, train, val, test = generate_dataloaders()
devices = [0, 1, 2, 3]
pad_idx = TGT.vocab.stoi["<blank>"]
print("Making model...")
model = make_model(len(SRC.vocab), len(TGT.vocab), N=6)
model.cuda()
criterion = LabelSmoothing(
size=len(TGT.vocab), padding_idx=pad_idx, smoothing=0.1)
criterion.cuda()
BATCH_SIZE = 12000
train_iter = BatchIterator(train, batch_size=BATCH_SIZE, device=torch.device(0),
repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn, train=True)
valid_iter = BatchIterator(val, batch_size=BATCH_SIZE, device=torch.device(0),
repeat=False, sort_key=lambda x: (len(x.src), len(x.trg)),
batch_size_fn=batch_size_fn, train=False)
model_par = nn.DataParallel(model, device_ids=devices)
model_opt = NoamOpt(model.src_embed[0].d_model, 1, 2000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
folder = get_unique_folder("./models/", "model")
if not(os.path.exists(folder)):
os.mkdir(folder)
for epoch in tqdm(range(10)):
model_par.train()
run_epoch((rebatch(pad_idx, b) for b in train_iter),
model_par,
MultiGPULossCompute(model.generator, criterion,
devices=devices, opt=model_opt))
model_par.eval()
loss = run_epoch((rebatch(pad_idx, b) for b in valid_iter),
model_par,
MultiGPULossCompute(model.generator, criterion,
devices=devices, opt=None))
torch.save(model.state_dict, os.path.join(folder, "model.bin." + str(epoch)))
print(loss)
for i, batch in enumerate(valid_iter):
src = batch.src.transpose(0, 1)[:1]
src_mask = (src != SRC.vocab.stoi["<blank>"]).unsqueeze(-2)
out = greedy_decode(model, src, src_mask,
max_len=60, start_symbol=TGT.vocab.stoi["<s>"])
print("Translation:", end="\t")
for i in range(1, out.size(1)):
sym = TGT.vocab.itos[out[0, i]]
if sym == "</s>":
break
print(sym, end=" ")
print()
print("Target:", end="\t")
for i in range(1, batch.trg.size(0)):
sym = TGT.vocab.itos[batch.trg.data[i, 0]]
if sym == "</s>":
break
print(sym, end=" ")
print()
break
def follower():
new_model = make_model()
dream_model = make_model()
while 1:
# print('waiting for packet')
packet = comm.recv(source=0)
#comm.Recv(packet, source=0)
current_env_name = packet['current_env_name']
dream_mode = packet['dream_mode']
packet = packet['result']
assert(len(packet) == SOLUTION_PACKET_SIZE), (len(packet), SOLUTION_PACKET_SIZE)
solutions = decode_solution_packet(packet)
results = []
if dream_mode:
new_model.make_env(current_env_name + '_dream', model = dream_model)
else:
new_model.make_env(current_env_name)
for i, solution in enumerate(solutions):
# print(f'working on solution {i+1} of {len(solutions)}')
worker_id, jobidx, seed, max_len, weights = solution
worker_id = int(worker_id)
possible_error = "work_id = " + str(worker_id) + " rank = " + str(rank)
assert worker_id == rank, possible_error
jobidx = int(jobidx)
seed = int(seed)
fitness, timesteps = worker(weights, seed, max_len, new_model)
results.append([worker_id, jobidx, fitness, timesteps])
new_model.env.close()
result_packet = encode_result_packet(results)
assert len(result_packet) == RESULT_PACKET_SIZE
comm.Send(result_packet, dest=0)
def __init__(self):
super().__init__()
self.net = make_model(conf["model"]).to(device=device)
self.net.load_weights(args)
self.renderer = NeRFRenderer.from_conf(
conf["renderer"],
white_bkgd=not args.black,
eval_batch_size=args.ray_batch_size).to(device=device)
def train(train_data, train_labels, file_name, NUM_EPOCHS=50, BATCH_SIZE=128, TRAIN_TEMP=1):
train_size = train_labels.shape[0]
train_xs = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
train_ys = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_LABELS))
model, saver = make_model(None, NUM_CHANNELS=NUM_CHANNELS, IMAGE_SIZE=IMAGE_SIZE)
logits = model(train_xs, True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits/TRAIN_TEMP,
labels=train_ys))
lr = tf.placeholder(tf.float32, [])
momentum = tf.placeholder(tf.float32, [])
optimizer = tf.train.MomentumOptimizer(lr, momentum).minimize(loss)
check_xs = tf.placeholder(tf.float32,
shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
check_prediction = tf.nn.softmax(model(check_xs))
def check(data, labels):
err = []
for i in range(0,len(data),100):
predicted = check_prediction.eval({check_xs: data[i:i+100]})
valids = np.argmax(predicted, 1) == np.argmax(labels[i:i+100], 1)
err.extend(valids)
return 1-np.mean(err)
with tf.Session() as s:
tf.global_variables_initializer().run()
for step in range(NUM_EPOCHS * train_size // BATCH_SIZE):
epoch = (float(step) * BATCH_SIZE / train_size)
offset = np.random.random_integers(0, train_size-1, BATCH_SIZE)
batch_data = train_data[offset, :, :, :]
batch_labels = train_labels[offset, :]
feed_dict = {train_xs: batch_data,
train_ys: batch_labels,
lr: (0.5**int(epoch/10))*0.01,
momentum: (0.5**int(epoch/10))*0.9}
s.run(optimizer, feed_dict=feed_dict)
if step % 100 == 0:
print("Step %d/%d"%(step, NUM_EPOCHS * train_size // BATCH_SIZE))
print("Validation error: ", check(validation_data, validation_labels))
print("\nTest error: ", check(test_data, test_labels))
saver(s, file_name)
def __init__(self, cfg):
self.model = make_model(cfg, 751)
self.model.load_param(cfg.TEST_WEIGHT)
self.device = "cuda"
if self.device:
if torch.cuda.device_count() > 1:
self.model = nn.DataParallel(self.model)
self.model.to(self.device)
self.model.eval()
def sample(args):
# load ascii text and covert to lowercase
with codecs.open(os.path.join(args.data_dir, 'input.txt'),
"r",
encoding='utf-8') as f:
raw_text = f.read()
# create mapping of unique chars to integers, and a reverse mapping
chars = sorted(list(set(raw_text)))
char_to_int = dict((c, i) for i, c in enumerate(chars))
int_to_char = dict((i, c) for i, c in enumerate(chars))
# summarize the loaded data
n_chars = len(raw_text)
n_vocab = len(chars)
print("Total Characters: ", n_chars)
print("Total Vocab: ", n_vocab)
# prepare the dataset of input to output pairs encoded as integers
seq_length = 100
dataX = []
dataY = []
for i in range(0, n_chars - seq_length, 1):
seq_in = raw_text[i:i + seq_length]
seq_out = raw_text[i + seq_length]
dataX.append([char_to_int[char] for char in seq_in])
dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print("Total Patterns: "), n_patterns
X = numpy.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# load the network weights
model = make_model(X, y)
iteration = 10
filename = find_file(
'snapshot_' + MODEL_NAME + '_{:02d}'.format(iteration) + '*.hdf5',
'models')
model.load_weights(filename)
model.compile(loss='categorical_crossentropy', optimizer='adam')
# pick a random seed
start = numpy.random.randint(0, len(dataX) - 1)
pattern = dataX[start]
print("Seed:")
print("\"", ''.join([int_to_char[value] for value in pattern]), "\"")
# generate characters
for i in range(1000):
x = numpy.reshape(pattern, (1, len(pattern), 1))
x = x / float(n_vocab)
prediction = model.predict(x, verbose=0)
index = numpy.argmax(prediction)
result = int_to_char[index]
seq_in = [int_to_char[value] for value in pattern]
sys.stdout.write(result)
pattern.append(index)
pattern = pattern[1:len(pattern)]
print("\nDone.")
def get_models(args):
ae_model = get_cuda(make_model(d_vocab=args.vocab_size,
N=args.num_layers_AE,
d_model=args.transformer_model_size,
latent_size=args.latent_size,
d_ff=args.transformer_ff_size))
dis_model = get_cuda(Classifier(latent_size=args.latent_size, output_size=args.label_size))
ae_model.load_state_dict(torch.load(args.current_save_path + 'ae_model_params.pkl', map_location=torch.device('cpu')))
dis_model.load_state_dict(torch.load(args.current_save_path + 'dis_model_params.pkl', map_location=torch.device('cpu')))
return ae_model, dis_model
def one_sentence_translate(sent, beam_search=True):
# 初始化模型
model = make_model(config.src_vocab_size, config.tgt_vocab_size,
config.n_layers, config.d_model, config.d_ff,
config.n_heads, config.dropout)
BOS = english_tokenizer_load().bos_id() # 2
EOS = english_tokenizer_load().eos_id() # 3
src_tokens = [[BOS] + english_tokenizer_load().EncodeAsIds(sent) + [EOS]]
batch_input = torch.LongTensor(np.array(src_tokens)).to(config.device)
translate(batch_input, model, use_beam=beam_search)
def main():
V = 11
model = make_model(V, V, N=2).to(device)
model.load_state_dict(torch.load('checkpoint/model.pt'))
model.to(device)
model.eval()
src = torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]).to(device)
src_mask = torch.ones(1, 1, 10).to(device)
result = greedy_decode(model, src, src_mask, max_len=10, start_symbol=1)
print(result)
def make_model_higher():
w_conv = randn(config.frame_out,
config.frame_len,
requires_grad=config.conv_deconv_grad)
if config.init_fourier:
with no_grad():
for f in range(config.frame_out):
w_conv[f, ...] = cos(2 * pi * (f + 1) / config.frame_len *
arange(0, config.frame_len, 1))
convolver = FF(w_conv)
else:
if config.init_xavier:
xavier_normal_(w_conv, gain=5 / 3)
convolver = FFT(w_conv)
if config.conv_deconv_same:
deconvolver = convolver
else:
if config.init_fourier:
w_deconv = w_conv.detach()
w_deconv.requires_grad = config.conv_deconv_grad
deconvolver = FF(w_deconv)
else:
w_deconv = randn(config.frame_out,
config.frame_len,
requires_grad=config.conv_deconv_grad)
if config.init_xavier:
xavier_normal_(w_deconv, gain=5 / 3)
deconvolver = FFT(w_deconv)
convolver = [convolver]
deconvolver = [deconvolver]
# body = config.creation_info[1:-1]
enc = make_model(config.attention1_info)
dec = make_model(config.attention2_info)
return [convolver, enc, dec, deconvolver]
def setup_embeddings(script, config):
# Get parameters
num_vertices = config.getint('Graph', 'num_vertices')
motif_size = config.getint('Motifs', 'motif_size')
walk_length = config.getint('Walks', 'walk_length')
embeddings_dir = config.get('Embeddings', 'embeddings_dir')
embed_dim = config.getint('Embeddings', 'embed_dim')
learn_rate = config.getfloat('Embeddings', 'learn_rate')
mini_batch_size = config.getint('Embeddings', 'mini_batch_size')
sgd_steps = config.getint('Embeddings', 'sgd_steps')
sgd_steps_per_epoch = config.getint('Embeddings', 'sgd_steps_per_epoch')
assert (num_vertices>0 and motif_size>0 and walk_length>=motif_size
and embeddings_dir and embed_dim>0 and mini_batch_size>0
and sgd_steps>=0 and sgd_steps_per_epoch>0), \
'invalid configuration for training embeddings'
# Construct LBANN objects
num_epochs = (sgd_steps + sgd_steps_per_epoch - 1) // sgd_steps_per_epoch
trainer = lbann.Trainer(
mini_batch_size=mini_batch_size,
num_parallel_readers=0,
)
model_ = make_model(
motif_size,
walk_length,
num_vertices,
embed_dim,
learn_rate,
num_epochs,
embeddings_dir,
)
optimizer = lbann.SGD(learn_rate=learn_rate)
data_reader = make_data_reader()
# Add LBANN invocation to batch script
prototext_file = os.path.join(script.work_dir, 'experiment.prototext')
lbann.proto.save_prototext(
prototext_file,
trainer=trainer,
model=model_,
data_reader=data_reader,
optimizer=optimizer,
)
script.add_body_line('')
script.add_body_line('# Train embeddings')
script.add_parallel_command([
lbann.lbann_exe(),
f'--prototext={prototext_file}',
f'--num_io_threads=1',
])
def check_opt():
"""check learning rate changes"""
import numpy as np
import matplotlib.pyplot as plt
model = make_model(config.src_vocab_size, config.tgt_vocab_size,
config.n_layers, config.d_model, config.d_ff,
config.n_heads, config.dropout)
opt = get_std_opt(model)
# Three settings of the lrate hyperparameters.
opts = [opt, NoamOpt(512, 1, 20000, None), NoamOpt(256, 1, 10000, None)]
plt.plot(np.arange(1, 50000),
[[opt.rate(i) for opt in opts] for i in range(1, 50000)])
plt.legend(["512:10000", "512:20000", "256:10000"])
plt.show()
def translate():
# Initialization
opt = parse_args()
src_word2idx = torch.load(opt.vocab)['src']
tgt_word2idx = torch.load(opt.vocab)['tgt']
print("src_vocab: {}, tgt_vocab: {}".format(len(src_word2idx),
len(tgt_word2idx)))
# Model definition
model = make_model(len(src_word2idx), len(tgt_word2idx))
translator = Translator(model, opt.beam_size, opt.n_best, opt.max_length)
translator.translate(opt)
def train(train_data, train_labels, teacher_name, file_name,
NUM_EPOCHS=50, BATCH_SIZE=128, TRAIN_TEMP=1):
# Step 1: train the teacher model
train_baseline.train(train_data, train_labels, teacher_name,
NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP)
# Step 2: evaluate the model on the training data at the training temperature
soft_labels = np.zeros(train_labels.shape)
with tf.Session() as s:
model = make_model(teacher_name)
xs = tf.placeholder(tf.float32,
shape=(100, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
predictions = tf.nn.softmax(model(xs)/TRAIN_TEMP)
for i in range(0,len(train_data),100):
predicted = predictions.eval({xs: train_data[i:i+100]})
soft_labels[i:i+100] = predicted
# Step 3: train the distilled model on the new training data
train_baseline.train(train_data, soft_labels, file_name,
NUM_EPOCHS, BATCH_SIZE, TRAIN_TEMP)
def slave():
new_model = make_model()
while 1:
#print('waiting for packet')
packet = comm.recv(source=0)
#comm.Recv(packet, source=0)
current_env_name = packet['current_env_name']
packet = packet['result']
assert(len(packet) == SOLUTION_PACKET_SIZE)
solutions = decode_solution_packet(packet)
results = []
#tracker2 = SummaryTracker()
new_model.make_env(current_env_name)
#tracker2.print_diff()
for solution in solutions:
worker_id, jobidx, seed, train_mode, max_len, weights = solution
assert (train_mode == 1 or train_mode == 0), str(train_mode)
worker_id = int(worker_id)
possible_error = "work_id = " + str(worker_id) + " rank = " + str(rank)
assert worker_id == rank, possible_error
jobidx = int(jobidx)
seed = int(seed)
fitness, timesteps = worker(weights, seed, max_len, new_model, train_mode)
results.append([worker_id, jobidx, fitness, timesteps])
new_model.env.close()
result_packet = encode_result_packet(results)
assert len(result_packet) == RESULT_PACKET_SIZE
comm.Send(result_packet, dest=0)
solver = KOSolver(k=k, n_t=n_t)
# Collect data
x = best.design.latin_center(num_samples, k)
Y = []
for i in range(num_samples):
Y.append(solver(x[i, :]))
X = (x, solver.X_fixed[0])
H = tuple([np.ones((x.shape[0], 1)) for x in X])
Y = np.vstack(Y)
# Save the data for later use
data_file = 'ko_data_s=%d.pickle' % num_samples
with open(data_file, 'wb') as fd:
pickle.dump((X, H, Y), fd, pickle.HIGHEST_PROTOCOL)
cgp_model = model.make_model(X, Y)
smc_sampler = best.smc.SMC(cgp_model, num_particles=num_particles,
num_mcmc=num_mcmc, verbose=1,
gamma_is_an_exponent=True,
mpi=mpi)
smc_sampler.initialize(0.)
pa = smc_sampler.get_particle_approximation()
smc_sampler.move_to(1.)
pa = smc_sampler.get_particle_approximation()
# dump the result to a file to process later
gpa = pa.allgather()
if mpi.COMM_WORLD.Get_rank() == 0:
out_file = 'ko_smc_s=%d_p=%d_m=%d.pickle' %(num_samples, num_particles,
num_mcmc)
from model import *
# from mpl_toolkits.basemap import Basemap
# from matplotlib import *
from pylab import *
from numpy import *
import model
import pymc as pm
data = csv2rec('duffy-jittered.csv')
# Convert from record array to dictionary
data = dict([(k,data[k]) for k in data.dtype.names])
# Create model. Use the HDF5 database backend, because the trace will take up a lot of memory.
DuffySampler = pm.MCMC(model.make_model(**data), db='hdf5', dbcomplevel=1, dbcomplib='zlib', dbname='duffydb.hdf5')
# ====================
# = Do the inference =
# ====================
# Use GPEvaluationGibbs step methods.
DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_b, DuffySampler.V_b, DuffySampler.tilde_fb)
DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_s, DuffySampler.V_s, DuffySampler.tilde_fs)
# Run the MCMC.
DuffySampler.isample(50000,10000,100)
n = len(DuffySampler.trace('V_b')[:])
# ==========================
# = Mean and variance maps =
import model
import best
import pymc
import matplotlib.pyplot as plt
if __name__ == '__main__':
# Set the sampling parameters
k = 1 # Number of dimensions
n_t = 10 # Number of time steps
num_samples = 20 # Number of samples
# Initialize the solver
solver = KOSolver(k=k, n_t=n_t)
# Collect data
X = best.design.latin_center(num_samples, k)
Y = []
for i in range(num_samples):
Y.append(solver(X[i, :]))
Y = np.vstack(Y)
cgp_model = model.make_model((X, solver.X_fixed[0]), Y)
mcmc_sampler = pymc.MCMC(cgp_model)
mcmc_sampler.sample(100000, thin=1000, burn=10000)
pymc.Matplot.plot(mcmc_sampler)
plt.show()
def initialize_settings(sigma_init=0.1, sigma_decay=0.9999, init_opt = ''):
global population, filebase, controller_filebase, model, num_params, es, PRECISION, SOLUTION_PACKET_SIZE, RESULT_PACKET_SIZE
population = num_worker * num_worker_trial
filebase = './log/'+env_name+'.'+optimizer+'.'+str(num_episode)+'.'+str(population)
controller_filebase = './controller/'+env_name+'.'+optimizer+'.'+str(num_episode)+'.'+str(population)
model = make_model()
num_params = model.param_count
#print("size of model", num_params)
if len(init_opt) > 0:
es = pickle.load(open(init_opt, 'rb'))
else:
if optimizer == 'ses':
ses = PEPG(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_alpha=0.2,
sigma_limit=0.02,
elite_ratio=0.1,
weight_decay=0.005,
popsize=population)
es = ses
elif optimizer == 'ga':
ga = SimpleGA(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_limit=0.02,
elite_ratio=0.1,
weight_decay=0.005,
popsize=population)
es = ga
elif optimizer == 'cma':
cma = CMAES(num_params,
sigma_init=sigma_init,
popsize=population)
es = cma
elif optimizer == 'pepg':
pepg = PEPG(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_alpha=0.20,
sigma_limit=0.02,
learning_rate=0.01,
learning_rate_decay=1.0,
learning_rate_limit=0.01,
weight_decay=0.005,
popsize=population)
es = pepg
else:
oes = OpenES(num_params,
sigma_init=sigma_init,
sigma_decay=sigma_decay,
sigma_limit=0.02,
learning_rate=0.01,
learning_rate_decay=1.0,
learning_rate_limit=0.01,
antithetic=antithetic,
weight_decay=0.005,
popsize=population)
es = oes
PRECISION = 10000
SOLUTION_PACKET_SIZE = (5+num_params)*num_worker_trial
RESULT_PACKET_SIZE = 4*num_worker_trial
from model import *
# from mpl_toolkits.basemap import Basemap
# from matplotlib import *
from pylab import *
from numpy import *
import model
import pymc as pm
data = csv2rec("duffy-jittered.csv")
# Convert from record array to dictionary
data = dict([(k, data[k]) for k in data.dtype.names])
# Create model. Use the HDF5 database backend, because the trace will take up a lot of memory.
DuffySampler = pm.MCMC(model.make_model(**data), db="hdf5", dbcomplevel=1, dbcomplib="zlib", dbname="duffydb.hdf5")
# ====================
# = Do the inference =
# ====================
# Use GPEvaluationGibbs step methods.
DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_b, DuffySampler.V_b, DuffySampler.tilde_fb)
DuffySampler.use_step_method(pm.gp.GPEvaluationGibbs, DuffySampler.sp_sub_s, DuffySampler.V_s, DuffySampler.tilde_fs)
# Run the MCMC.
DuffySampler.isample(50000, 10000, 100)
n = len(DuffySampler.trace("V_b")[:])
# ==========================
# = Mean and variance maps =
import sys
import time
import tensorflow as tf
import numpy as np
from model import make_model, preprocess
from setup import *
def compute(dat,lab):
BATCH_SIZE = 100
img = tf.Variable(np.zeros((BATCH_SIZE,IMAGE_SIZE,IMAGE_SIZE,NUM_CHANNELS),dtype=np.float32))
out = model(img)
r = []
for i in range(0,len(dat),BATCH_SIZE):
data = img
o = s.run(out, feed_dict={img: preprocess(dat[i:i+BATCH_SIZE])})
r.extend(o)
print(np.mean(np.abs(r)),np.std(np.abs(r)))
with tf.Session() as s:
model = make_model(sys.argv[1],s)
compute(test_data, test_labels)
